Bug Summary

File:lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Warning:line 6722, column 14
Forming reference to null pointer

Annotated Source Code

1//===-- SelectionDAGBuilder.cpp - Selection-DAG building ------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements routines for translating from LLVM IR into SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SelectionDAGBuilder.h"
15#include "SDNodeDbgValue.h"
16#include "llvm/ADT/BitVector.h"
17#include "llvm/ADT/Optional.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/ADT/Statistic.h"
20#include "llvm/Analysis/AliasAnalysis.h"
21#include "llvm/Analysis/BranchProbabilityInfo.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/Loads.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/Analysis/VectorUtils.h"
27#include "llvm/CodeGen/Analysis.h"
28#include "llvm/CodeGen/FastISel.h"
29#include "llvm/CodeGen/FunctionLoweringInfo.h"
30#include "llvm/CodeGen/GCMetadata.h"
31#include "llvm/CodeGen/GCStrategy.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineJumpTableInfo.h"
36#include "llvm/CodeGen/MachineModuleInfo.h"
37#include "llvm/CodeGen/MachineRegisterInfo.h"
38#include "llvm/CodeGen/SelectionDAG.h"
39#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
40#include "llvm/CodeGen/StackMaps.h"
41#include "llvm/CodeGen/WinEHFuncInfo.h"
42#include "llvm/IR/CallingConv.h"
43#include "llvm/IR/ConstantRange.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DebugInfo.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/GetElementPtrTypeIterator.h"
50#include "llvm/IR/GlobalVariable.h"
51#include "llvm/IR/InlineAsm.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/IntrinsicInst.h"
54#include "llvm/IR/Intrinsics.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Statepoint.h"
58#include "llvm/MC/MCSymbol.h"
59#include "llvm/Support/CommandLine.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/MathExtras.h"
63#include "llvm/Support/raw_ostream.h"
64#include "llvm/Target/TargetFrameLowering.h"
65#include "llvm/Target/TargetInstrInfo.h"
66#include "llvm/Target/TargetIntrinsicInfo.h"
67#include "llvm/Target/TargetLowering.h"
68#include "llvm/Target/TargetOptions.h"
69#include "llvm/Target/TargetSubtargetInfo.h"
70#include <algorithm>
71#include <utility>
72using namespace llvm;
73
74#define DEBUG_TYPE"isel" "isel"
75
76/// LimitFloatPrecision - Generate low-precision inline sequences for
77/// some float libcalls (6, 8 or 12 bits).
78static unsigned LimitFloatPrecision;
79
80static cl::opt<unsigned, true>
81LimitFPPrecision("limit-float-precision",
82 cl::desc("Generate low-precision inline sequences "
83 "for some float libcalls"),
84 cl::location(LimitFloatPrecision),
85 cl::init(0));
86// Limit the width of DAG chains. This is important in general to prevent
87// DAG-based analysis from blowing up. For example, alias analysis and
88// load clustering may not complete in reasonable time. It is difficult to
89// recognize and avoid this situation within each individual analysis, and
90// future analyses are likely to have the same behavior. Limiting DAG width is
91// the safe approach and will be especially important with global DAGs.
92//
93// MaxParallelChains default is arbitrarily high to avoid affecting
94// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
95// sequence over this should have been converted to llvm.memcpy by the
96// frontend. It is easy to induce this behavior with .ll code such as:
97// %buffer = alloca [4096 x i8]
98// %data = load [4096 x i8]* %argPtr
99// store [4096 x i8] %data, [4096 x i8]* %buffer
100static const unsigned MaxParallelChains = 64;
101
102static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
103 const SDValue *Parts, unsigned NumParts,
104 MVT PartVT, EVT ValueVT, const Value *V);
105
106/// getCopyFromParts - Create a value that contains the specified legal parts
107/// combined into the value they represent. If the parts combine to a type
108/// larger than ValueVT then AssertOp can be used to specify whether the extra
109/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
110/// (ISD::AssertSext).
111static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
112 const SDValue *Parts, unsigned NumParts,
113 MVT PartVT, EVT ValueVT, const Value *V,
114 Optional<ISD::NodeType> AssertOp = None) {
115 if (ValueVT.isVector())
116 return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
117 PartVT, ValueVT, V);
118
119 assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 119, __PRETTY_FUNCTION__))
;
120 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
121 SDValue Val = Parts[0];
122
123 if (NumParts > 1) {
124 // Assemble the value from multiple parts.
125 if (ValueVT.isInteger()) {
126 unsigned PartBits = PartVT.getSizeInBits();
127 unsigned ValueBits = ValueVT.getSizeInBits();
128
129 // Assemble the power of 2 part.
130 unsigned RoundParts = NumParts & (NumParts - 1) ?
131 1 << Log2_32(NumParts) : NumParts;
132 unsigned RoundBits = PartBits * RoundParts;
133 EVT RoundVT = RoundBits == ValueBits ?
134 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
135 SDValue Lo, Hi;
136
137 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
138
139 if (RoundParts > 2) {
140 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
141 PartVT, HalfVT, V);
142 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
143 RoundParts / 2, PartVT, HalfVT, V);
144 } else {
145 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
146 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
147 }
148
149 if (DAG.getDataLayout().isBigEndian())
150 std::swap(Lo, Hi);
151
152 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
153
154 if (RoundParts < NumParts) {
155 // Assemble the trailing non-power-of-2 part.
156 unsigned OddParts = NumParts - RoundParts;
157 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
158 Hi = getCopyFromParts(DAG, DL,
159 Parts + RoundParts, OddParts, PartVT, OddVT, V);
160
161 // Combine the round and odd parts.
162 Lo = Val;
163 if (DAG.getDataLayout().isBigEndian())
164 std::swap(Lo, Hi);
165 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
166 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
167 Hi =
168 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
169 DAG.getConstant(Lo.getValueSizeInBits(), DL,
170 TLI.getPointerTy(DAG.getDataLayout())));
171 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
172 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
173 }
174 } else if (PartVT.isFloatingPoint()) {
175 // FP split into multiple FP parts (for ppcf128)
176 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 177, __PRETTY_FUNCTION__))
177 "Unexpected split")((ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
"Unexpected split") ? static_cast<void> (0) : __assert_fail
("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 177, __PRETTY_FUNCTION__))
;
178 SDValue Lo, Hi;
179 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
180 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
181 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
182 std::swap(Lo, Hi);
183 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
184 } else {
185 // FP split into integer parts (soft fp)
186 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 187, __PRETTY_FUNCTION__))
187 !PartVT.isVector() && "Unexpected split")((ValueVT.isFloatingPoint() && PartVT.isInteger() &&
!PartVT.isVector() && "Unexpected split") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 187, __PRETTY_FUNCTION__))
;
188 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
189 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
190 }
191 }
192
193 // There is now one part, held in Val. Correct it to match ValueVT.
194 // PartEVT is the type of the register class that holds the value.
195 // ValueVT is the type of the inline asm operation.
196 EVT PartEVT = Val.getValueType();
197
198 if (PartEVT == ValueVT)
199 return Val;
200
201 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
202 ValueVT.bitsLT(PartEVT)) {
203 // For an FP value in an integer part, we need to truncate to the right
204 // width first.
205 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
206 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
207 }
208
209 // Handle types that have the same size.
210 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
211 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
212
213 // Handle types with different sizes.
214 if (PartEVT.isInteger() && ValueVT.isInteger()) {
215 if (ValueVT.bitsLT(PartEVT)) {
216 // For a truncate, see if we have any information to
217 // indicate whether the truncated bits will always be
218 // zero or sign-extension.
219 if (AssertOp.hasValue())
220 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
221 DAG.getValueType(ValueVT));
222 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
223 }
224 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
225 }
226
227 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
228 // FP_ROUND's are always exact here.
229 if (ValueVT.bitsLT(Val.getValueType()))
230 return DAG.getNode(
231 ISD::FP_ROUND, DL, ValueVT, Val,
232 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
233
234 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
235 }
236
237 llvm_unreachable("Unknown mismatch!")::llvm::llvm_unreachable_internal("Unknown mismatch!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 237)
;
238}
239
240static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
241 const Twine &ErrMsg) {
242 const Instruction *I = dyn_cast_or_null<Instruction>(V);
243 if (!V)
244 return Ctx.emitError(ErrMsg);
245
246 const char *AsmError = ", possible invalid constraint for vector type";
247 if (const CallInst *CI = dyn_cast<CallInst>(I))
248 if (isa<InlineAsm>(CI->getCalledValue()))
249 return Ctx.emitError(I, ErrMsg + AsmError);
250
251 return Ctx.emitError(I, ErrMsg);
252}
253
254/// getCopyFromPartsVector - Create a value that contains the specified legal
255/// parts combined into the value they represent. If the parts combine to a
256/// type larger than ValueVT then AssertOp can be used to specify whether the
257/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
258/// ValueVT (ISD::AssertSext).
259static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
260 const SDValue *Parts, unsigned NumParts,
261 MVT PartVT, EVT ValueVT, const Value *V) {
262 assert(ValueVT.isVector() && "Not a vector value")((ValueVT.isVector() && "Not a vector value") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 262, __PRETTY_FUNCTION__))
;
263 assert(NumParts > 0 && "No parts to assemble!")((NumParts > 0 && "No parts to assemble!") ? static_cast
<void> (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 263, __PRETTY_FUNCTION__))
;
264 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
265 SDValue Val = Parts[0];
266
267 // Handle a multi-element vector.
268 if (NumParts > 1) {
269 EVT IntermediateVT;
270 MVT RegisterVT;
271 unsigned NumIntermediates;
272 unsigned NumRegs =
273 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
274 NumIntermediates, RegisterVT);
275 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 275, __PRETTY_FUNCTION__))
;
276 NumParts = NumRegs; // Silence a compiler warning.
277 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 277, __PRETTY_FUNCTION__))
;
278 assert(RegisterVT.getSizeInBits() ==((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 280, __PRETTY_FUNCTION__))
279 Parts[0].getSimpleValueType().getSizeInBits() &&((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 280, __PRETTY_FUNCTION__))
280 "Part type sizes don't match!")((RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType()
.getSizeInBits() && "Part type sizes don't match!") ?
static_cast<void> (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 280, __PRETTY_FUNCTION__))
;
281
282 // Assemble the parts into intermediate operands.
283 SmallVector<SDValue, 8> Ops(NumIntermediates);
284 if (NumIntermediates == NumParts) {
285 // If the register was not expanded, truncate or copy the value,
286 // as appropriate.
287 for (unsigned i = 0; i != NumParts; ++i)
288 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
289 PartVT, IntermediateVT, V);
290 } else if (NumParts > 0) {
291 // If the intermediate type was expanded, build the intermediate
292 // operands from the parts.
293 assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 294, __PRETTY_FUNCTION__))
294 "Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 294, __PRETTY_FUNCTION__))
;
295 unsigned Factor = NumParts / NumIntermediates;
296 for (unsigned i = 0; i != NumIntermediates; ++i)
297 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
298 PartVT, IntermediateVT, V);
299 }
300
301 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
302 // intermediate operands.
303 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
304 : ISD::BUILD_VECTOR,
305 DL, ValueVT, Ops);
306 }
307
308 // There is now one part, held in Val. Correct it to match ValueVT.
309 EVT PartEVT = Val.getValueType();
310
311 if (PartEVT == ValueVT)
312 return Val;
313
314 if (PartEVT.isVector()) {
315 // If the element type of the source/dest vectors are the same, but the
316 // parts vector has more elements than the value vector, then we have a
317 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
318 // elements we want.
319 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
320 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&((PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements
() && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 321, __PRETTY_FUNCTION__))
321 "Cannot narrow, it would be a lossy transformation")((PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements
() && "Cannot narrow, it would be a lossy transformation"
) ? static_cast<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 321, __PRETTY_FUNCTION__))
;
322 return DAG.getNode(
323 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
324 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
325 }
326
327 // Vector/Vector bitcast.
328 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
329 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
330
331 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&((PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 332, __PRETTY_FUNCTION__))
332 "Cannot handle this kind of promotion")((PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements
() && "Cannot handle this kind of promotion") ? static_cast
<void> (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 332, __PRETTY_FUNCTION__))
;
333 // Promoted vector extract
334 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
335
336 }
337
338 // Trivial bitcast if the types are the same size and the destination
339 // vector type is legal.
340 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
341 TLI.isTypeLegal(ValueVT))
342 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
343
344 // Handle cases such as i8 -> <1 x i1>
345 if (ValueVT.getVectorNumElements() != 1) {
346 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
347 "non-trivial scalar-to-vector conversion");
348 return DAG.getUNDEF(ValueVT);
349 }
350
351 EVT ValueSVT = ValueVT.getVectorElementType();
352 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
353 Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
354 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
355
356 return DAG.getBuildVector(ValueVT, DL, Val);
357}
358
359static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
360 SDValue Val, SDValue *Parts, unsigned NumParts,
361 MVT PartVT, const Value *V);
362
363/// getCopyToParts - Create a series of nodes that contain the specified value
364/// split into legal parts. If the parts contain more bits than Val, then, for
365/// integers, ExtendKind can be used to specify how to generate the extra bits.
366static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
367 SDValue *Parts, unsigned NumParts, MVT PartVT,
368 const Value *V,
369 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
370 EVT ValueVT = Val.getValueType();
371
372 // Handle the vector case separately.
373 if (ValueVT.isVector())
374 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
375
376 unsigned PartBits = PartVT.getSizeInBits();
377 unsigned OrigNumParts = NumParts;
378 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 379, __PRETTY_FUNCTION__))
379 "Copying to an illegal type!")((DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && "Copying to an illegal type!"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 379, __PRETTY_FUNCTION__))
;
380
381 if (NumParts == 0)
382 return;
383
384 assert(!ValueVT.isVector() && "Vector case handled elsewhere")((!ValueVT.isVector() && "Vector case handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 384, __PRETTY_FUNCTION__))
;
385 EVT PartEVT = PartVT;
386 if (PartEVT == ValueVT) {
387 assert(NumParts == 1 && "No-op copy with multiple parts!")((NumParts == 1 && "No-op copy with multiple parts!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 387, __PRETTY_FUNCTION__))
;
388 Parts[0] = Val;
389 return;
390 }
391
392 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
393 // If the parts cover more bits than the value has, promote the value.
394 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
395 assert(NumParts == 1 && "Do not know what to promote to!")((NumParts == 1 && "Do not know what to promote to!")
? static_cast<void> (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 395, __PRETTY_FUNCTION__))
;
396 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
397 } else {
398 if (ValueVT.isFloatingPoint()) {
399 // FP values need to be bitcast, then extended if they are being put
400 // into a larger container.
401 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
402 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
403 }
404 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 406, __PRETTY_FUNCTION__))
405 ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 406, __PRETTY_FUNCTION__))
406 "Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 406, __PRETTY_FUNCTION__))
;
407 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
408 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
409 if (PartVT == MVT::x86mmx)
410 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
411 }
412 } else if (PartBits == ValueVT.getSizeInBits()) {
413 // Different types of the same size.
414 assert(NumParts == 1 && PartEVT != ValueVT)((NumParts == 1 && PartEVT != ValueVT) ? static_cast<
void> (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 414, __PRETTY_FUNCTION__))
;
415 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
416 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
417 // If the parts cover less bits than value has, truncate the value.
418 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 420, __PRETTY_FUNCTION__))
419 ValueVT.isInteger() &&(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 420, __PRETTY_FUNCTION__))
420 "Unknown mismatch!")(((PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT
.isInteger() && "Unknown mismatch!") ? static_cast<
void> (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 420, __PRETTY_FUNCTION__))
;
421 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
422 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
423 if (PartVT == MVT::x86mmx)
424 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
425 }
426
427 // The value may have changed - recompute ValueVT.
428 ValueVT = Val.getValueType();
429 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 430, __PRETTY_FUNCTION__))
430 "Failed to tile the value with PartVT!")((NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"
) ? static_cast<void> (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 430, __PRETTY_FUNCTION__))
;
431
432 if (NumParts == 1) {
433 if (PartEVT != ValueVT) {
434 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
435 "scalar-to-vector conversion failed");
436 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
437 }
438
439 Parts[0] = Val;
440 return;
441 }
442
443 // Expand the value into multiple parts.
444 if (NumParts & (NumParts - 1)) {
445 // The number of parts is not a power of 2. Split off and copy the tail.
446 assert(PartVT.isInteger() && ValueVT.isInteger() &&((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 447, __PRETTY_FUNCTION__))
447 "Do not know what to expand to!")((PartVT.isInteger() && ValueVT.isInteger() &&
"Do not know what to expand to!") ? static_cast<void> (
0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 447, __PRETTY_FUNCTION__))
;
448 unsigned RoundParts = 1 << Log2_32(NumParts);
449 unsigned RoundBits = RoundParts * PartBits;
450 unsigned OddParts = NumParts - RoundParts;
451 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
452 DAG.getIntPtrConstant(RoundBits, DL));
453 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
454
455 if (DAG.getDataLayout().isBigEndian())
456 // The odd parts were reversed by getCopyToParts - unreverse them.
457 std::reverse(Parts + RoundParts, Parts + NumParts);
458
459 NumParts = RoundParts;
460 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
461 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
462 }
463
464 // The number of parts is a power of 2. Repeatedly bisect the value using
465 // EXTRACT_ELEMENT.
466 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
467 EVT::getIntegerVT(*DAG.getContext(),
468 ValueVT.getSizeInBits()),
469 Val);
470
471 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
472 for (unsigned i = 0; i < NumParts; i += StepSize) {
473 unsigned ThisBits = StepSize * PartBits / 2;
474 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
475 SDValue &Part0 = Parts[i];
476 SDValue &Part1 = Parts[i+StepSize/2];
477
478 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
479 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
480 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
481 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
482
483 if (ThisBits == PartBits && ThisVT != PartVT) {
484 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
485 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
486 }
487 }
488 }
489
490 if (DAG.getDataLayout().isBigEndian())
491 std::reverse(Parts, Parts + OrigNumParts);
492}
493
494
495/// getCopyToPartsVector - Create a series of nodes that contain the specified
496/// value split into legal parts.
497static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
498 SDValue Val, SDValue *Parts, unsigned NumParts,
499 MVT PartVT, const Value *V) {
500 EVT ValueVT = Val.getValueType();
501 assert(ValueVT.isVector() && "Not a vector")((ValueVT.isVector() && "Not a vector") ? static_cast
<void> (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __PRETTY_FUNCTION__))
;
502 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
503
504 if (NumParts == 1) {
505 EVT PartEVT = PartVT;
506 if (PartEVT == ValueVT) {
507 // Nothing to do.
508 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
509 // Bitconvert vector->vector case.
510 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
511 } else if (PartVT.isVector() &&
512 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
513 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
514 EVT ElementVT = PartVT.getVectorElementType();
515 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
516 // undef elements.
517 SmallVector<SDValue, 16> Ops;
518 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
519 Ops.push_back(DAG.getNode(
520 ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
521 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
522
523 for (unsigned i = ValueVT.getVectorNumElements(),
524 e = PartVT.getVectorNumElements(); i != e; ++i)
525 Ops.push_back(DAG.getUNDEF(ElementVT));
526
527 Val = DAG.getBuildVector(PartVT, DL, Ops);
528
529 // FIXME: Use CONCAT for 2x -> 4x.
530
531 //SDValue UndefElts = DAG.getUNDEF(VectorTy);
532 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
533 } else if (PartVT.isVector() &&
534 PartEVT.getVectorElementType().bitsGE(
535 ValueVT.getVectorElementType()) &&
536 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
537
538 // Promoted vector extract
539 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
540 } else{
541 // Vector -> scalar conversion.
542 assert(ValueVT.getVectorNumElements() == 1 &&((ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"
) ? static_cast<void> (0) : __assert_fail ("ValueVT.getVectorNumElements() == 1 && \"Only trivial vector-to-scalar conversions should get here!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 543, __PRETTY_FUNCTION__))
543 "Only trivial vector-to-scalar conversions should get here!")((ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"
) ? static_cast<void> (0) : __assert_fail ("ValueVT.getVectorNumElements() == 1 && \"Only trivial vector-to-scalar conversions should get here!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 543, __PRETTY_FUNCTION__))
;
544 Val = DAG.getNode(
545 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
546 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
547 }
548
549 assert(Val.getValueType() == PartVT && "Unexpected vector part value type")((Val.getValueType() == PartVT && "Unexpected vector part value type"
) ? static_cast<void> (0) : __assert_fail ("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 549, __PRETTY_FUNCTION__))
;
550 Parts[0] = Val;
551 return;
552 }
553
554 // Handle a multi-element vector.
555 EVT IntermediateVT;
556 MVT RegisterVT;
557 unsigned NumIntermediates;
558 unsigned NumRegs = TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT,
559 IntermediateVT,
560 NumIntermediates, RegisterVT);
561 unsigned NumElements = ValueVT.getVectorNumElements();
562
563 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")((NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 563, __PRETTY_FUNCTION__))
;
564 NumParts = NumRegs; // Silence a compiler warning.
565 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")((RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? static_cast<void> (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 565, __PRETTY_FUNCTION__))
;
566
567 // Split the vector into intermediate operands.
568 SmallVector<SDValue, 8> Ops(NumIntermediates);
569 for (unsigned i = 0; i != NumIntermediates; ++i) {
570 if (IntermediateVT.isVector())
571 Ops[i] =
572 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
573 DAG.getConstant(i * (NumElements / NumIntermediates), DL,
574 TLI.getVectorIdxTy(DAG.getDataLayout())));
575 else
576 Ops[i] = DAG.getNode(
577 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
578 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
579 }
580
581 // Split the intermediate operands into legal parts.
582 if (NumParts == NumIntermediates) {
583 // If the register was not expanded, promote or copy the value,
584 // as appropriate.
585 for (unsigned i = 0; i != NumParts; ++i)
586 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
587 } else if (NumParts > 0) {
588 // If the intermediate type was expanded, split each the value into
589 // legal parts.
590 assert(NumIntermediates != 0 && "division by zero")((NumIntermediates != 0 && "division by zero") ? static_cast
<void> (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 590, __PRETTY_FUNCTION__))
;
591 assert(NumParts % NumIntermediates == 0 &&((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 592, __PRETTY_FUNCTION__))
592 "Must expand into a divisible number of parts!")((NumParts % NumIntermediates == 0 && "Must expand into a divisible number of parts!"
) ? static_cast<void> (0) : __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 592, __PRETTY_FUNCTION__))
;
593 unsigned Factor = NumParts / NumIntermediates;
594 for (unsigned i = 0; i != NumIntermediates; ++i)
595 getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
596 }
597}
598
599RegsForValue::RegsForValue() {}
600
601RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
602 EVT valuevt)
603 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {}
604
605RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
606 const DataLayout &DL, unsigned Reg, Type *Ty) {
607 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
608
609 for (EVT ValueVT : ValueVTs) {
610 unsigned NumRegs = TLI.getNumRegisters(Context, ValueVT);
611 MVT RegisterVT = TLI.getRegisterType(Context, ValueVT);
612 for (unsigned i = 0; i != NumRegs; ++i)
613 Regs.push_back(Reg + i);
614 RegVTs.push_back(RegisterVT);
615 Reg += NumRegs;
616 }
617}
618
619SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
620 FunctionLoweringInfo &FuncInfo,
621 const SDLoc &dl, SDValue &Chain,
622 SDValue *Flag, const Value *V) const {
623 // A Value with type {} or [0 x %t] needs no registers.
624 if (ValueVTs.empty())
625 return SDValue();
626
627 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
628
629 // Assemble the legal parts into the final values.
630 SmallVector<SDValue, 4> Values(ValueVTs.size());
631 SmallVector<SDValue, 8> Parts;
632 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
633 // Copy the legal parts from the registers.
634 EVT ValueVT = ValueVTs[Value];
635 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
636 MVT RegisterVT = RegVTs[Value];
637
638 Parts.resize(NumRegs);
639 for (unsigned i = 0; i != NumRegs; ++i) {
640 SDValue P;
641 if (!Flag) {
642 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
643 } else {
644 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
645 *Flag = P.getValue(2);
646 }
647
648 Chain = P.getValue(1);
649 Parts[i] = P;
650
651 // If the source register was virtual and if we know something about it,
652 // add an assert node.
653 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
654 !RegisterVT.isInteger() || RegisterVT.isVector())
655 continue;
656
657 const FunctionLoweringInfo::LiveOutInfo *LOI =
658 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
659 if (!LOI)
660 continue;
661
662 unsigned RegSize = RegisterVT.getSizeInBits();
663 unsigned NumSignBits = LOI->NumSignBits;
664 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
665
666 if (NumZeroBits == RegSize) {
667 // The current value is a zero.
668 // Explicitly express that as it would be easier for
669 // optimizations to kick in.
670 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
671 continue;
672 }
673
674 // FIXME: We capture more information than the dag can represent. For
675 // now, just use the tightest assertzext/assertsext possible.
676 bool isSExt = true;
677 EVT FromVT(MVT::Other);
678 if (NumSignBits == RegSize) {
679 isSExt = true; // ASSERT SEXT 1
680 FromVT = MVT::i1;
681 } else if (NumZeroBits >= RegSize - 1) {
682 isSExt = false; // ASSERT ZEXT 1
683 FromVT = MVT::i1;
684 } else if (NumSignBits > RegSize - 8) {
685 isSExt = true; // ASSERT SEXT 8
686 FromVT = MVT::i8;
687 } else if (NumZeroBits >= RegSize - 8) {
688 isSExt = false; // ASSERT ZEXT 8
689 FromVT = MVT::i8;
690 } else if (NumSignBits > RegSize - 16) {
691 isSExt = true; // ASSERT SEXT 16
692 FromVT = MVT::i16;
693 } else if (NumZeroBits >= RegSize - 16) {
694 isSExt = false; // ASSERT ZEXT 16
695 FromVT = MVT::i16;
696 } else if (NumSignBits > RegSize - 32) {
697 isSExt = true; // ASSERT SEXT 32
698 FromVT = MVT::i32;
699 } else if (NumZeroBits >= RegSize - 32) {
700 isSExt = false; // ASSERT ZEXT 32
701 FromVT = MVT::i32;
702 } else {
703 continue;
704 }
705 // Add an assertion node.
706 assert(FromVT != MVT::Other)((FromVT != MVT::Other) ? static_cast<void> (0) : __assert_fail
("FromVT != MVT::Other", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 706, __PRETTY_FUNCTION__))
;
707 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
708 RegisterVT, P, DAG.getValueType(FromVT));
709 }
710
711 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
712 NumRegs, RegisterVT, ValueVT, V);
713 Part += NumRegs;
714 Parts.clear();
715 }
716
717 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
718}
719
720void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
721 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
722 const Value *V,
723 ISD::NodeType PreferredExtendType) const {
724 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
725 ISD::NodeType ExtendKind = PreferredExtendType;
726
727 // Get the list of the values's legal parts.
728 unsigned NumRegs = Regs.size();
729 SmallVector<SDValue, 8> Parts(NumRegs);
730 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
731 EVT ValueVT = ValueVTs[Value];
732 unsigned NumParts = TLI.getNumRegisters(*DAG.getContext(), ValueVT);
733 MVT RegisterVT = RegVTs[Value];
734
735 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
736 ExtendKind = ISD::ZERO_EXTEND;
737
738 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
739 &Parts[Part], NumParts, RegisterVT, V, ExtendKind);
740 Part += NumParts;
741 }
742
743 // Copy the parts into the registers.
744 SmallVector<SDValue, 8> Chains(NumRegs);
745 for (unsigned i = 0; i != NumRegs; ++i) {
746 SDValue Part;
747 if (!Flag) {
748 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
749 } else {
750 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
751 *Flag = Part.getValue(1);
752 }
753
754 Chains[i] = Part.getValue(0);
755 }
756
757 if (NumRegs == 1 || Flag)
758 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
759 // flagged to it. That is the CopyToReg nodes and the user are considered
760 // a single scheduling unit. If we create a TokenFactor and return it as
761 // chain, then the TokenFactor is both a predecessor (operand) of the
762 // user as well as a successor (the TF operands are flagged to the user).
763 // c1, f1 = CopyToReg
764 // c2, f2 = CopyToReg
765 // c3 = TokenFactor c1, c2
766 // ...
767 // = op c3, ..., f2
768 Chain = Chains[NumRegs-1];
769 else
770 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
771}
772
773void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
774 unsigned MatchingIdx, const SDLoc &dl,
775 SelectionDAG &DAG,
776 std::vector<SDValue> &Ops) const {
777 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
778
779 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
780 if (HasMatching)
781 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
782 else if (!Regs.empty() &&
783 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
784 // Put the register class of the virtual registers in the flag word. That
785 // way, later passes can recompute register class constraints for inline
786 // assembly as well as normal instructions.
787 // Don't do this for tied operands that can use the regclass information
788 // from the def.
789 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
790 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
791 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
792 }
793
794 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
795 Ops.push_back(Res);
796
797 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
798 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
799 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
800 MVT RegisterVT = RegVTs[Value];
801 for (unsigned i = 0; i != NumRegs; ++i) {
802 assert(Reg < Regs.size() && "Mismatch in # registers expected")((Reg < Regs.size() && "Mismatch in # registers expected"
) ? static_cast<void> (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 802, __PRETTY_FUNCTION__))
;
803 unsigned TheReg = Regs[Reg++];
804 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
805
806 if (TheReg == SP && Code == InlineAsm::Kind_Clobber) {
807 // If we clobbered the stack pointer, MFI should know about it.
808 assert(DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment())((DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment
()) ? static_cast<void> (0) : __assert_fail ("DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 808, __PRETTY_FUNCTION__))
;
809 }
810 }
811 }
812}
813
814void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
815 const TargetLibraryInfo *li) {
816 AA = aa;
817 GFI = gfi;
818 LibInfo = li;
819 DL = &DAG.getDataLayout();
820 Context = DAG.getContext();
821 LPadToCallSiteMap.clear();
822}
823
824void SelectionDAGBuilder::clear() {
825 NodeMap.clear();
826 UnusedArgNodeMap.clear();
827 PendingLoads.clear();
828 PendingExports.clear();
829 CurInst = nullptr;
830 HasTailCall = false;
831 SDNodeOrder = LowestSDNodeOrder;
832 StatepointLowering.clear();
833}
834
835void SelectionDAGBuilder::clearDanglingDebugInfo() {
836 DanglingDebugInfoMap.clear();
837}
838
839SDValue SelectionDAGBuilder::getRoot() {
840 if (PendingLoads.empty())
841 return DAG.getRoot();
842
843 if (PendingLoads.size() == 1) {
844 SDValue Root = PendingLoads[0];
845 DAG.setRoot(Root);
846 PendingLoads.clear();
847 return Root;
848 }
849
850 // Otherwise, we have to make a token factor node.
851 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
852 PendingLoads);
853 PendingLoads.clear();
854 DAG.setRoot(Root);
855 return Root;
856}
857
858SDValue SelectionDAGBuilder::getControlRoot() {
859 SDValue Root = DAG.getRoot();
860
861 if (PendingExports.empty())
862 return Root;
863
864 // Turn all of the CopyToReg chains into one factored node.
865 if (Root.getOpcode() != ISD::EntryToken) {
866 unsigned i = 0, e = PendingExports.size();
867 for (; i != e; ++i) {
868 assert(PendingExports[i].getNode()->getNumOperands() > 1)((PendingExports[i].getNode()->getNumOperands() > 1) ? static_cast
<void> (0) : __assert_fail ("PendingExports[i].getNode()->getNumOperands() > 1"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 868, __PRETTY_FUNCTION__))
;
869 if (PendingExports[i].getNode()->getOperand(0) == Root)
870 break; // Don't add the root if we already indirectly depend on it.
871 }
872
873 if (i == e)
874 PendingExports.push_back(Root);
875 }
876
877 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
878 PendingExports);
879 PendingExports.clear();
880 DAG.setRoot(Root);
881 return Root;
882}
883
884void SelectionDAGBuilder::visit(const Instruction &I) {
885 // Set up outgoing PHI node register values before emitting the terminator.
886 if (isa<TerminatorInst>(&I)) {
887 HandlePHINodesInSuccessorBlocks(I.getParent());
888 }
889
890 // Increase the SDNodeOrder if dealing with a non-debug instruction.
891 if (!isa<DbgInfoIntrinsic>(I))
892 ++SDNodeOrder;
893
894 CurInst = &I;
895
896 visit(I.getOpcode(), I);
897
898 if (!isa<TerminatorInst>(&I) && !HasTailCall &&
899 !isStatepoint(&I)) // statepoints handle their exports internally
900 CopyToExportRegsIfNeeded(&I);
901
902 CurInst = nullptr;
903}
904
905void SelectionDAGBuilder::visitPHI(const PHINode &) {
906 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 906)
;
907}
908
909void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
910 // Note: this doesn't use InstVisitor, because it has to work with
911 // ConstantExpr's in addition to instructions.
912 switch (Opcode) {
913 default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 913)
;
914 // Build the switch statement using the Instruction.def file.
915#define HANDLE_INST(NUM, OPCODE, CLASS) \
916 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
917#include "llvm/IR/Instruction.def"
918 }
919}
920
921// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
922// generate the debug data structures now that we've seen its definition.
923void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
924 SDValue Val) {
925 DanglingDebugInfo &DDI = DanglingDebugInfoMap[V];
926 if (DDI.getDI()) {
927 const DbgValueInst *DI = DDI.getDI();
928 DebugLoc dl = DDI.getdl();
929 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
930 DILocalVariable *Variable = DI->getVariable();
931 DIExpression *Expr = DI->getExpression();
932 assert(Variable->isValidLocationForIntrinsic(dl) &&((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 933, __PRETTY_FUNCTION__))
933 "Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(dl) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 933, __PRETTY_FUNCTION__))
;
934 uint64_t Offset = DI->getOffset();
935 SDDbgValue *SDV;
936 if (Val.getNode()) {
937 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, Offset, false,
938 Val)) {
939 SDV = getDbgValue(Val, Variable, Expr, Offset, dl, DbgSDNodeOrder);
940 DAG.AddDbgValue(SDV, Val.getNode(), false);
941 }
942 } else
943 DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false)
;
944 DanglingDebugInfoMap[V] = DanglingDebugInfo();
945 }
946}
947
948/// getCopyFromRegs - If there was virtual register allocated for the value V
949/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
950SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
951 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
952 SDValue Result;
953
954 if (It != FuncInfo.ValueMap.end()) {
955 unsigned InReg = It->second;
956 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
957 DAG.getDataLayout(), InReg, Ty);
958 SDValue Chain = DAG.getEntryNode();
959 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
960 resolveDanglingDebugInfo(V, Result);
961 }
962
963 return Result;
964}
965
966/// getValue - Return an SDValue for the given Value.
967SDValue SelectionDAGBuilder::getValue(const Value *V) {
968 // If we already have an SDValue for this value, use it. It's important
969 // to do this first, so that we don't create a CopyFromReg if we already
970 // have a regular SDValue.
971 SDValue &N = NodeMap[V];
972 if (N.getNode()) return N;
973
974 // If there's a virtual register allocated and initialized for this
975 // value, use it.
976 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
977 return copyFromReg;
978
979 // Otherwise create a new SDValue and remember it.
980 SDValue Val = getValueImpl(V);
981 NodeMap[V] = Val;
982 resolveDanglingDebugInfo(V, Val);
983 return Val;
984}
985
986// Return true if SDValue exists for the given Value
987bool SelectionDAGBuilder::findValue(const Value *V) const {
988 return (NodeMap.find(V) != NodeMap.end()) ||
989 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
990}
991
992/// getNonRegisterValue - Return an SDValue for the given Value, but
993/// don't look in FuncInfo.ValueMap for a virtual register.
994SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
995 // If we already have an SDValue for this value, use it.
996 SDValue &N = NodeMap[V];
997 if (N.getNode()) {
998 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
999 // Remove the debug location from the node as the node is about to be used
1000 // in a location which may differ from the original debug location. This
1001 // is relevant to Constant and ConstantFP nodes because they can appear
1002 // as constant expressions inside PHI nodes.
1003 N->setDebugLoc(DebugLoc());
1004 }
1005 return N;
1006 }
1007
1008 // Otherwise create a new SDValue and remember it.
1009 SDValue Val = getValueImpl(V);
1010 NodeMap[V] = Val;
1011 resolveDanglingDebugInfo(V, Val);
1012 return Val;
1013}
1014
1015/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1016/// Create an SDValue for the given value.
1017SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1018 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1019
1020 if (const Constant *C = dyn_cast<Constant>(V)) {
1021 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1022
1023 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1024 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1025
1026 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1027 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1028
1029 if (isa<ConstantPointerNull>(C)) {
1030 unsigned AS = V->getType()->getPointerAddressSpace();
1031 return DAG.getConstant(0, getCurSDLoc(),
1032 TLI.getPointerTy(DAG.getDataLayout(), AS));
1033 }
1034
1035 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1036 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1037
1038 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1039 return DAG.getUNDEF(VT);
1040
1041 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1042 visit(CE->getOpcode(), *CE);
1043 SDValue N1 = NodeMap[V];
1044 assert(N1.getNode() && "visit didn't populate the NodeMap!")((N1.getNode() && "visit didn't populate the NodeMap!"
) ? static_cast<void> (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1044, __PRETTY_FUNCTION__))
;
1045 return N1;
1046 }
1047
1048 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1049 SmallVector<SDValue, 4> Constants;
1050 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1051 OI != OE; ++OI) {
1052 SDNode *Val = getValue(*OI).getNode();
1053 // If the operand is an empty aggregate, there are no values.
1054 if (!Val) continue;
1055 // Add each leaf value from the operand to the Constants list
1056 // to form a flattened list of all the values.
1057 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1058 Constants.push_back(SDValue(Val, i));
1059 }
1060
1061 return DAG.getMergeValues(Constants, getCurSDLoc());
1062 }
1063
1064 if (const ConstantDataSequential *CDS =
1065 dyn_cast<ConstantDataSequential>(C)) {
1066 SmallVector<SDValue, 4> Ops;
1067 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1068 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1069 // Add each leaf value from the operand to the Constants list
1070 // to form a flattened list of all the values.
1071 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1072 Ops.push_back(SDValue(Val, i));
1073 }
1074
1075 if (isa<ArrayType>(CDS->getType()))
1076 return DAG.getMergeValues(Ops, getCurSDLoc());
1077 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1078 }
1079
1080 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1081 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1082, __PRETTY_FUNCTION__))
1082 "Unknown struct or array constant!")(((isa<ConstantAggregateZero>(C) || isa<UndefValue>
(C)) && "Unknown struct or array constant!") ? static_cast
<void> (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1082, __PRETTY_FUNCTION__))
;
1083
1084 SmallVector<EVT, 4> ValueVTs;
1085 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1086 unsigned NumElts = ValueVTs.size();
1087 if (NumElts == 0)
1088 return SDValue(); // empty struct
1089 SmallVector<SDValue, 4> Constants(NumElts);
1090 for (unsigned i = 0; i != NumElts; ++i) {
1091 EVT EltVT = ValueVTs[i];
1092 if (isa<UndefValue>(C))
1093 Constants[i] = DAG.getUNDEF(EltVT);
1094 else if (EltVT.isFloatingPoint())
1095 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1096 else
1097 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1098 }
1099
1100 return DAG.getMergeValues(Constants, getCurSDLoc());
1101 }
1102
1103 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1104 return DAG.getBlockAddress(BA, VT);
1105
1106 VectorType *VecTy = cast<VectorType>(V->getType());
1107 unsigned NumElements = VecTy->getNumElements();
1108
1109 // Now that we know the number and type of the elements, get that number of
1110 // elements into the Ops array based on what kind of constant it is.
1111 SmallVector<SDValue, 16> Ops;
1112 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1113 for (unsigned i = 0; i != NumElements; ++i)
1114 Ops.push_back(getValue(CV->getOperand(i)));
1115 } else {
1116 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!")((isa<ConstantAggregateZero>(C) && "Unknown vector constant!"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantAggregateZero>(C) && \"Unknown vector constant!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1116, __PRETTY_FUNCTION__))
;
1117 EVT EltVT =
1118 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1119
1120 SDValue Op;
1121 if (EltVT.isFloatingPoint())
1122 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1123 else
1124 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1125 Ops.assign(NumElements, Op);
1126 }
1127
1128 // Create a BUILD_VECTOR node.
1129 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1130 }
1131
1132 // If this is a static alloca, generate it as the frameindex instead of
1133 // computation.
1134 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1135 DenseMap<const AllocaInst*, int>::iterator SI =
1136 FuncInfo.StaticAllocaMap.find(AI);
1137 if (SI != FuncInfo.StaticAllocaMap.end())
1138 return DAG.getFrameIndex(SI->second,
1139 TLI.getFrameIndexTy(DAG.getDataLayout()));
1140 }
1141
1142 // If this is an instruction which fast-isel has deferred, select it now.
1143 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1144 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1145 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1146 Inst->getType());
1147 SDValue Chain = DAG.getEntryNode();
1148 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1149 }
1150
1151 llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1151)
;
1152}
1153
1154void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1155 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1156 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1157 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1158 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1159 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1160 if (IsMSVCCXX || IsCoreCLR)
1161 CatchPadMBB->setIsEHFuncletEntry();
1162
1163 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other, getControlRoot()));
1164}
1165
1166void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1167 // Update machine-CFG edge.
1168 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1169 FuncInfo.MBB->addSuccessor(TargetMBB);
1170
1171 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1172 bool IsSEH = isAsynchronousEHPersonality(Pers);
1173 if (IsSEH) {
1174 // If this is not a fall-through branch or optimizations are switched off,
1175 // emit the branch.
1176 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1177 TM.getOptLevel() == CodeGenOpt::None)
1178 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1179 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1180 return;
1181 }
1182
1183 // Figure out the funclet membership for the catchret's successor.
1184 // This will be used by the FuncletLayout pass to determine how to order the
1185 // BB's.
1186 // A 'catchret' returns to the outer scope's color.
1187 Value *ParentPad = I.getCatchSwitchParentPad();
1188 const BasicBlock *SuccessorColor;
1189 if (isa<ConstantTokenNone>(ParentPad))
1190 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1191 else
1192 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1193 assert(SuccessorColor && "No parent funclet for catchret!")((SuccessorColor && "No parent funclet for catchret!"
) ? static_cast<void> (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1193, __PRETTY_FUNCTION__))
;
1194 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1195 assert(SuccessorColorMBB && "No MBB for SuccessorColor!")((SuccessorColorMBB && "No MBB for SuccessorColor!") ?
static_cast<void> (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1195, __PRETTY_FUNCTION__))
;
1196
1197 // Create the terminator node.
1198 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1199 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1200 DAG.getBasicBlock(SuccessorColorMBB));
1201 DAG.setRoot(Ret);
1202}
1203
1204void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1205 // Don't emit any special code for the cleanuppad instruction. It just marks
1206 // the start of a funclet.
1207 FuncInfo.MBB->setIsEHFuncletEntry();
1208 FuncInfo.MBB->setIsCleanupFuncletEntry();
1209}
1210
1211/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1212/// many places it could ultimately go. In the IR, we have a single unwind
1213/// destination, but in the machine CFG, we enumerate all the possible blocks.
1214/// This function skips over imaginary basic blocks that hold catchswitch
1215/// instructions, and finds all the "real" machine
1216/// basic block destinations. As those destinations may not be successors of
1217/// EHPadBB, here we also calculate the edge probability to those destinations.
1218/// The passed-in Prob is the edge probability to EHPadBB.
1219static void findUnwindDestinations(
1220 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1221 BranchProbability Prob,
1222 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1223 &UnwindDests) {
1224 EHPersonality Personality =
1225 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1226 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1227 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1228
1229 while (EHPadBB) {
1230 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1231 BasicBlock *NewEHPadBB = nullptr;
1232 if (isa<LandingPadInst>(Pad)) {
1233 // Stop on landingpads. They are not funclets.
1234 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1235 break;
1236 } else if (isa<CleanupPadInst>(Pad)) {
1237 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1238 // personalities.
1239 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1240 UnwindDests.back().first->setIsEHFuncletEntry();
1241 break;
1242 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1243 // Add the catchpad handlers to the possible destinations.
1244 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1245 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1246 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1247 if (IsMSVCCXX || IsCoreCLR)
1248 UnwindDests.back().first->setIsEHFuncletEntry();
1249 }
1250 NewEHPadBB = CatchSwitch->getUnwindDest();
1251 } else {
1252 continue;
1253 }
1254
1255 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1256 if (BPI && NewEHPadBB)
1257 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1258 EHPadBB = NewEHPadBB;
1259 }
1260}
1261
1262void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1263 // Update successor info.
1264 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1265 auto UnwindDest = I.getUnwindDest();
1266 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1267 BranchProbability UnwindDestProb =
1268 (BPI && UnwindDest)
1269 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1270 : BranchProbability::getZero();
1271 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1272 for (auto &UnwindDest : UnwindDests) {
1273 UnwindDest.first->setIsEHPad();
1274 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1275 }
1276 FuncInfo.MBB->normalizeSuccProbs();
1277
1278 // Create the terminator node.
1279 SDValue Ret =
1280 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1281 DAG.setRoot(Ret);
1282}
1283
1284void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1285 report_fatal_error("visitCatchSwitch not yet implemented!");
1286}
1287
1288void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1289 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1290 auto &DL = DAG.getDataLayout();
1291 SDValue Chain = getControlRoot();
1292 SmallVector<ISD::OutputArg, 8> Outs;
1293 SmallVector<SDValue, 8> OutVals;
1294
1295 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1296 // lower
1297 //
1298 // %val = call <ty> @llvm.experimental.deoptimize()
1299 // ret <ty> %val
1300 //
1301 // differently.
1302 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1303 LowerDeoptimizingReturn();
1304 return;
1305 }
1306
1307 if (!FuncInfo.CanLowerReturn) {
1308 unsigned DemoteReg = FuncInfo.DemoteRegister;
1309 const Function *F = I.getParent()->getParent();
1310
1311 // Emit a store of the return value through the virtual register.
1312 // Leave Outs empty so that LowerReturn won't try to load return
1313 // registers the usual way.
1314 SmallVector<EVT, 1> PtrValueVTs;
1315 ComputeValueVTs(TLI, DL, PointerType::getUnqual(F->getReturnType()),
1316 PtrValueVTs);
1317
1318 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1319 DemoteReg, PtrValueVTs[0]);
1320 SDValue RetOp = getValue(I.getOperand(0));
1321
1322 SmallVector<EVT, 4> ValueVTs;
1323 SmallVector<uint64_t, 4> Offsets;
1324 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1325 unsigned NumValues = ValueVTs.size();
1326
1327 // An aggregate return value cannot wrap around the address space, so
1328 // offsets to its parts don't wrap either.
1329 SDNodeFlags Flags;
1330 Flags.setNoUnsignedWrap(true);
1331
1332 SmallVector<SDValue, 4> Chains(NumValues);
1333 for (unsigned i = 0; i != NumValues; ++i) {
1334 SDValue Add = DAG.getNode(ISD::ADD, getCurSDLoc(),
1335 RetPtr.getValueType(), RetPtr,
1336 DAG.getIntPtrConstant(Offsets[i],
1337 getCurSDLoc()),
1338 Flags);
1339 Chains[i] = DAG.getStore(Chain, getCurSDLoc(),
1340 SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1341 // FIXME: better loc info would be nice.
1342 Add, MachinePointerInfo());
1343 }
1344
1345 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1346 MVT::Other, Chains);
1347 } else if (I.getNumOperands() != 0) {
1348 SmallVector<EVT, 4> ValueVTs;
1349 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1350 unsigned NumValues = ValueVTs.size();
1351 if (NumValues) {
1352 SDValue RetOp = getValue(I.getOperand(0));
1353
1354 const Function *F = I.getParent()->getParent();
1355
1356 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1357 if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1358 Attribute::SExt))
1359 ExtendKind = ISD::SIGN_EXTEND;
1360 else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1361 Attribute::ZExt))
1362 ExtendKind = ISD::ZERO_EXTEND;
1363
1364 LLVMContext &Context = F->getContext();
1365 bool RetInReg = F->getAttributes().hasAttribute(
1366 AttributeList::ReturnIndex, Attribute::InReg);
1367
1368 for (unsigned j = 0; j != NumValues; ++j) {
1369 EVT VT = ValueVTs[j];
1370
1371 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1372 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1373
1374 unsigned NumParts = TLI.getNumRegisters(Context, VT);
1375 MVT PartVT = TLI.getRegisterType(Context, VT);
1376 SmallVector<SDValue, 4> Parts(NumParts);
1377 getCopyToParts(DAG, getCurSDLoc(),
1378 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1379 &Parts[0], NumParts, PartVT, &I, ExtendKind);
1380
1381 // 'inreg' on function refers to return value
1382 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1383 if (RetInReg)
1384 Flags.setInReg();
1385
1386 // Propagate extension type if any
1387 if (ExtendKind == ISD::SIGN_EXTEND)
1388 Flags.setSExt();
1389 else if (ExtendKind == ISD::ZERO_EXTEND)
1390 Flags.setZExt();
1391
1392 for (unsigned i = 0; i < NumParts; ++i) {
1393 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1394 VT, /*isfixed=*/true, 0, 0));
1395 OutVals.push_back(Parts[i]);
1396 }
1397 }
1398 }
1399 }
1400
1401 // Push in swifterror virtual register as the last element of Outs. This makes
1402 // sure swifterror virtual register will be returned in the swifterror
1403 // physical register.
1404 const Function *F = I.getParent()->getParent();
1405 if (TLI.supportSwiftError() &&
1406 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1407 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument")((FuncInfo.SwiftErrorArg && "Need a swift error argument"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.SwiftErrorArg && \"Need a swift error argument\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1407, __PRETTY_FUNCTION__))
;
1408 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1409 Flags.setSwiftError();
1410 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1411 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1412 true /*isfixed*/, 1 /*origidx*/,
1413 0 /*partOffs*/));
1414 // Create SDNode for the swifterror virtual register.
1415 OutVals.push_back(DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVReg(
1416 FuncInfo.MBB, FuncInfo.SwiftErrorArg),
1417 EVT(TLI.getPointerTy(DL))));
1418 }
1419
1420 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1421 CallingConv::ID CallConv =
1422 DAG.getMachineFunction().getFunction()->getCallingConv();
1423 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1424 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1425
1426 // Verify that the target's LowerReturn behaved as expected.
1427 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1428, __PRETTY_FUNCTION__))
1428 "LowerReturn didn't return a valid chain!")((Chain.getNode() && Chain.getValueType() == MVT::Other
&& "LowerReturn didn't return a valid chain!") ? static_cast
<void> (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1428, __PRETTY_FUNCTION__))
;
1429
1430 // Update the DAG with the new chain value resulting from return lowering.
1431 DAG.setRoot(Chain);
1432}
1433
1434/// CopyToExportRegsIfNeeded - If the given value has virtual registers
1435/// created for it, emit nodes to copy the value into the virtual
1436/// registers.
1437void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1438 // Skip empty types
1439 if (V->getType()->isEmptyTy())
1440 return;
1441
1442 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1443 if (VMI != FuncInfo.ValueMap.end()) {
1444 assert(!V->use_empty() && "Unused value assigned virtual registers!")((!V->use_empty() && "Unused value assigned virtual registers!"
) ? static_cast<void> (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1444, __PRETTY_FUNCTION__))
;
1445 CopyValueToVirtualRegister(V, VMI->second);
1446 }
1447}
1448
1449/// ExportFromCurrentBlock - If this condition isn't known to be exported from
1450/// the current basic block, add it to ValueMap now so that we'll get a
1451/// CopyTo/FromReg.
1452void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1453 // No need to export constants.
1454 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1455
1456 // Already exported?
1457 if (FuncInfo.isExportedInst(V)) return;
1458
1459 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1460 CopyValueToVirtualRegister(V, Reg);
1461}
1462
1463bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1464 const BasicBlock *FromBB) {
1465 // The operands of the setcc have to be in this block. We don't know
1466 // how to export them from some other block.
1467 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1468 // Can export from current BB.
1469 if (VI->getParent() == FromBB)
1470 return true;
1471
1472 // Is already exported, noop.
1473 return FuncInfo.isExportedInst(V);
1474 }
1475
1476 // If this is an argument, we can export it if the BB is the entry block or
1477 // if it is already exported.
1478 if (isa<Argument>(V)) {
1479 if (FromBB == &FromBB->getParent()->getEntryBlock())
1480 return true;
1481
1482 // Otherwise, can only export this if it is already exported.
1483 return FuncInfo.isExportedInst(V);
1484 }
1485
1486 // Otherwise, constants can always be exported.
1487 return true;
1488}
1489
1490/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1491BranchProbability
1492SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1493 const MachineBasicBlock *Dst) const {
1494 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1495 const BasicBlock *SrcBB = Src->getBasicBlock();
1496 const BasicBlock *DstBB = Dst->getBasicBlock();
1497 if (!BPI) {
1498 // If BPI is not available, set the default probability as 1 / N, where N is
1499 // the number of successors.
1500 auto SuccSize = std::max<uint32_t>(
1501 std::distance(succ_begin(SrcBB), succ_end(SrcBB)), 1);
1502 return BranchProbability(1, SuccSize);
1503 }
1504 return BPI->getEdgeProbability(SrcBB, DstBB);
1505}
1506
1507void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1508 MachineBasicBlock *Dst,
1509 BranchProbability Prob) {
1510 if (!FuncInfo.BPI)
1511 Src->addSuccessorWithoutProb(Dst);
1512 else {
1513 if (Prob.isUnknown())
1514 Prob = getEdgeProbability(Src, Dst);
1515 Src->addSuccessor(Dst, Prob);
1516 }
1517}
1518
1519static bool InBlock(const Value *V, const BasicBlock *BB) {
1520 if (const Instruction *I = dyn_cast<Instruction>(V))
1521 return I->getParent() == BB;
1522 return true;
1523}
1524
1525/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1526/// This function emits a branch and is used at the leaves of an OR or an
1527/// AND operator tree.
1528///
1529void
1530SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1531 MachineBasicBlock *TBB,
1532 MachineBasicBlock *FBB,
1533 MachineBasicBlock *CurBB,
1534 MachineBasicBlock *SwitchBB,
1535 BranchProbability TProb,
1536 BranchProbability FProb,
1537 bool InvertCond) {
1538 const BasicBlock *BB = CurBB->getBasicBlock();
1539
1540 // If the leaf of the tree is a comparison, merge the condition into
1541 // the caseblock.
1542 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1543 // The operands of the cmp have to be in this block. We don't know
1544 // how to export them from some other block. If this is the first block
1545 // of the sequence, no exporting is needed.
1546 if (CurBB == SwitchBB ||
1547 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1548 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1549 ISD::CondCode Condition;
1550 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1551 ICmpInst::Predicate Pred =
1552 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1553 Condition = getICmpCondCode(Pred);
1554 } else {
1555 const FCmpInst *FC = cast<FCmpInst>(Cond);
1556 FCmpInst::Predicate Pred =
1557 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1558 Condition = getFCmpCondCode(Pred);
1559 if (TM.Options.NoNaNsFPMath)
1560 Condition = getFCmpCodeWithoutNaN(Condition);
1561 }
1562
1563 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1564 TBB, FBB, CurBB, TProb, FProb);
1565 SwitchCases.push_back(CB);
1566 return;
1567 }
1568 }
1569
1570 // Create a CaseBlock record representing this branch.
1571 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1572 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1573 nullptr, TBB, FBB, CurBB, TProb, FProb);
1574 SwitchCases.push_back(CB);
1575}
1576
1577/// FindMergedConditions - If Cond is an expression like
1578void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1579 MachineBasicBlock *TBB,
1580 MachineBasicBlock *FBB,
1581 MachineBasicBlock *CurBB,
1582 MachineBasicBlock *SwitchBB,
1583 Instruction::BinaryOps Opc,
1584 BranchProbability TProb,
1585 BranchProbability FProb,
1586 bool InvertCond) {
1587 // Skip over not part of the tree and remember to invert op and operands at
1588 // next level.
1589 if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1590 const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1591 if (InBlock(CondOp, CurBB->getBasicBlock())) {
1592 FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1593 !InvertCond);
1594 return;
1595 }
1596 }
1597
1598 const Instruction *BOp = dyn_cast<Instruction>(Cond);
1599 // Compute the effective opcode for Cond, taking into account whether it needs
1600 // to be inverted, e.g.
1601 // and (not (or A, B)), C
1602 // gets lowered as
1603 // and (and (not A, not B), C)
1604 unsigned BOpc = 0;
1605 if (BOp) {
1606 BOpc = BOp->getOpcode();
1607 if (InvertCond) {
1608 if (BOpc == Instruction::And)
1609 BOpc = Instruction::Or;
1610 else if (BOpc == Instruction::Or)
1611 BOpc = Instruction::And;
1612 }
1613 }
1614
1615 // If this node is not part of the or/and tree, emit it as a branch.
1616 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1617 BOpc != Opc || !BOp->hasOneUse() ||
1618 BOp->getParent() != CurBB->getBasicBlock() ||
1619 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1620 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1621 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1622 TProb, FProb, InvertCond);
1623 return;
1624 }
1625
1626 // Create TmpBB after CurBB.
1627 MachineFunction::iterator BBI(CurBB);
1628 MachineFunction &MF = DAG.getMachineFunction();
1629 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1630 CurBB->getParent()->insert(++BBI, TmpBB);
1631
1632 if (Opc == Instruction::Or) {
1633 // Codegen X | Y as:
1634 // BB1:
1635 // jmp_if_X TBB
1636 // jmp TmpBB
1637 // TmpBB:
1638 // jmp_if_Y TBB
1639 // jmp FBB
1640 //
1641
1642 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1643 // The requirement is that
1644 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1645 // = TrueProb for original BB.
1646 // Assuming the original probabilities are A and B, one choice is to set
1647 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1648 // A/(1+B) and 2B/(1+B). This choice assumes that
1649 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1650 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1651 // TmpBB, but the math is more complicated.
1652
1653 auto NewTrueProb = TProb / 2;
1654 auto NewFalseProb = TProb / 2 + FProb;
1655 // Emit the LHS condition.
1656 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1657 NewTrueProb, NewFalseProb, InvertCond);
1658
1659 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1660 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1661 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1662 // Emit the RHS condition into TmpBB.
1663 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1664 Probs[0], Probs[1], InvertCond);
1665 } else {
1666 assert(Opc == Instruction::And && "Unknown merge op!")((Opc == Instruction::And && "Unknown merge op!") ? static_cast
<void> (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1666, __PRETTY_FUNCTION__))
;
1667 // Codegen X & Y as:
1668 // BB1:
1669 // jmp_if_X TmpBB
1670 // jmp FBB
1671 // TmpBB:
1672 // jmp_if_Y TBB
1673 // jmp FBB
1674 //
1675 // This requires creation of TmpBB after CurBB.
1676
1677 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1678 // The requirement is that
1679 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1680 // = FalseProb for original BB.
1681 // Assuming the original probabilities are A and B, one choice is to set
1682 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1683 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1684 // TrueProb for BB1 * FalseProb for TmpBB.
1685
1686 auto NewTrueProb = TProb + FProb / 2;
1687 auto NewFalseProb = FProb / 2;
1688 // Emit the LHS condition.
1689 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1690 NewTrueProb, NewFalseProb, InvertCond);
1691
1692 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1693 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1694 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1695 // Emit the RHS condition into TmpBB.
1696 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1697 Probs[0], Probs[1], InvertCond);
1698 }
1699}
1700
1701/// If the set of cases should be emitted as a series of branches, return true.
1702/// If we should emit this as a bunch of and/or'd together conditions, return
1703/// false.
1704bool
1705SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1706 if (Cases.size() != 2) return true;
1707
1708 // If this is two comparisons of the same values or'd or and'd together, they
1709 // will get folded into a single comparison, so don't emit two blocks.
1710 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1711 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1712 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1713 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1714 return false;
1715 }
1716
1717 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1718 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1719 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1720 Cases[0].CC == Cases[1].CC &&
1721 isa<Constant>(Cases[0].CmpRHS) &&
1722 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1723 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1724 return false;
1725 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1726 return false;
1727 }
1728
1729 return true;
1730}
1731
1732void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1733 MachineBasicBlock *BrMBB = FuncInfo.MBB;
1734
1735 // Update machine-CFG edges.
1736 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1737
1738 if (I.isUnconditional()) {
1739 // Update machine-CFG edges.
1740 BrMBB->addSuccessor(Succ0MBB);
1741
1742 // If this is not a fall-through branch or optimizations are switched off,
1743 // emit the branch.
1744 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1745 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1746 MVT::Other, getControlRoot(),
1747 DAG.getBasicBlock(Succ0MBB)));
1748
1749 return;
1750 }
1751
1752 // If this condition is one of the special cases we handle, do special stuff
1753 // now.
1754 const Value *CondVal = I.getCondition();
1755 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1756
1757 // If this is a series of conditions that are or'd or and'd together, emit
1758 // this as a sequence of branches instead of setcc's with and/or operations.
1759 // As long as jumps are not expensive, this should improve performance.
1760 // For example, instead of something like:
1761 // cmp A, B
1762 // C = seteq
1763 // cmp D, E
1764 // F = setle
1765 // or C, F
1766 // jnz foo
1767 // Emit:
1768 // cmp A, B
1769 // je foo
1770 // cmp D, E
1771 // jle foo
1772 //
1773 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
1774 Instruction::BinaryOps Opcode = BOp->getOpcode();
1775 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
1776 !I.getMetadata(LLVMContext::MD_unpredictable) &&
1777 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
1778 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
1779 Opcode,
1780 getEdgeProbability(BrMBB, Succ0MBB),
1781 getEdgeProbability(BrMBB, Succ1MBB),
1782 /*InvertCond=*/false);
1783 // If the compares in later blocks need to use values not currently
1784 // exported from this block, export them now. This block should always
1785 // be the first entry.
1786 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")((SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"
) ? static_cast<void> (0) : __assert_fail ("SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1786, __PRETTY_FUNCTION__))
;
1787
1788 // Allow some cases to be rejected.
1789 if (ShouldEmitAsBranches(SwitchCases)) {
1790 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
1791 ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
1792 ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
1793 }
1794
1795 // Emit the branch for this block.
1796 visitSwitchCase(SwitchCases[0], BrMBB);
1797 SwitchCases.erase(SwitchCases.begin());
1798 return;
1799 }
1800
1801 // Okay, we decided not to do this, remove any inserted MBB's and clear
1802 // SwitchCases.
1803 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
1804 FuncInfo.MF->erase(SwitchCases[i].ThisBB);
1805
1806 SwitchCases.clear();
1807 }
1808 }
1809
1810 // Create a CaseBlock record representing this branch.
1811 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
1812 nullptr, Succ0MBB, Succ1MBB, BrMBB);
1813
1814 // Use visitSwitchCase to actually insert the fast branch sequence for this
1815 // cond branch.
1816 visitSwitchCase(CB, BrMBB);
1817}
1818
1819/// visitSwitchCase - Emits the necessary code to represent a single node in
1820/// the binary search tree resulting from lowering a switch instruction.
1821void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
1822 MachineBasicBlock *SwitchBB) {
1823 SDValue Cond;
1824 SDValue CondLHS = getValue(CB.CmpLHS);
1825 SDLoc dl = getCurSDLoc();
1826
1827 // Build the setcc now.
1828 if (!CB.CmpMHS) {
1829 // Fold "(X == true)" to X and "(X == false)" to !X to
1830 // handle common cases produced by branch lowering.
1831 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
1832 CB.CC == ISD::SETEQ)
1833 Cond = CondLHS;
1834 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
1835 CB.CC == ISD::SETEQ) {
1836 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
1837 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
1838 } else
1839 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
1840 } else {
1841 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")((CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? static_cast<void> (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1841, __PRETTY_FUNCTION__))
;
1842
1843 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
1844 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
1845
1846 SDValue CmpOp = getValue(CB.CmpMHS);
1847 EVT VT = CmpOp.getValueType();
1848
1849 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
1850 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
1851 ISD::SETLE);
1852 } else {
1853 SDValue SUB = DAG.getNode(ISD::SUB, dl,
1854 VT, CmpOp, DAG.getConstant(Low, dl, VT));
1855 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
1856 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
1857 }
1858 }
1859
1860 // Update successor info
1861 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
1862 // TrueBB and FalseBB are always different unless the incoming IR is
1863 // degenerate. This only happens when running llc on weird IR.
1864 if (CB.TrueBB != CB.FalseBB)
1865 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
1866 SwitchBB->normalizeSuccProbs();
1867
1868 // If the lhs block is the next block, invert the condition so that we can
1869 // fall through to the lhs instead of the rhs block.
1870 if (CB.TrueBB == NextBlock(SwitchBB)) {
1871 std::swap(CB.TrueBB, CB.FalseBB);
1872 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
1873 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
1874 }
1875
1876 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1877 MVT::Other, getControlRoot(), Cond,
1878 DAG.getBasicBlock(CB.TrueBB));
1879
1880 // Insert the false branch. Do this even if it's a fall through branch,
1881 // this makes it easier to do DAG optimizations which require inverting
1882 // the branch condition.
1883 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1884 DAG.getBasicBlock(CB.FalseBB));
1885
1886 DAG.setRoot(BrCond);
1887}
1888
1889/// visitJumpTable - Emit JumpTable node in the current MBB
1890void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
1891 // Emit the code for the jump table
1892 assert(JT.Reg != -1U && "Should lower JT Header first!")((JT.Reg != -1U && "Should lower JT Header first!") ?
static_cast<void> (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1892, __PRETTY_FUNCTION__))
;
1893 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1894 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
1895 JT.Reg, PTy);
1896 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
1897 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
1898 MVT::Other, Index.getValue(1),
1899 Table, Index);
1900 DAG.setRoot(BrJumpTable);
1901}
1902
1903/// visitJumpTableHeader - This function emits necessary code to produce index
1904/// in the JumpTable from switch case.
1905void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
1906 JumpTableHeader &JTH,
1907 MachineBasicBlock *SwitchBB) {
1908 SDLoc dl = getCurSDLoc();
1909
1910 // Subtract the lowest switch case value from the value being switched on and
1911 // conditional branch to default mbb if the result is greater than the
1912 // difference between smallest and largest cases.
1913 SDValue SwitchOp = getValue(JTH.SValue);
1914 EVT VT = SwitchOp.getValueType();
1915 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
1916 DAG.getConstant(JTH.First, dl, VT));
1917
1918 // The SDNode we just created, which holds the value being switched on minus
1919 // the smallest case value, needs to be copied to a virtual register so it
1920 // can be used as an index into the jump table in a subsequent basic block.
1921 // This value may be smaller or larger than the target's pointer type, and
1922 // therefore require extension or truncating.
1923 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1924 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
1925
1926 unsigned JumpTableReg =
1927 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
1928 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
1929 JumpTableReg, SwitchOp);
1930 JT.Reg = JumpTableReg;
1931
1932 // Emit the range check for the jump table, and branch to the default block
1933 // for the switch statement if the value being switched on exceeds the largest
1934 // case in the switch.
1935 SDValue CMP = DAG.getSetCC(
1936 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1937 Sub.getValueType()),
1938 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
1939
1940 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
1941 MVT::Other, CopyTo, CMP,
1942 DAG.getBasicBlock(JT.Default));
1943
1944 // Avoid emitting unnecessary branches to the next block.
1945 if (JT.MBB != NextBlock(SwitchBB))
1946 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
1947 DAG.getBasicBlock(JT.MBB));
1948
1949 DAG.setRoot(BrCond);
1950}
1951
1952/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
1953/// variable if there exists one.
1954static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
1955 SDValue &Chain) {
1956 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1957 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
1958 MachineFunction &MF = DAG.getMachineFunction();
1959 Value *Global = TLI.getSDagStackGuard(*MF.getFunction()->getParent());
1960 MachineSDNode *Node =
1961 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
1962 if (Global) {
1963 MachinePointerInfo MPInfo(Global);
1964 MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
1965 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1966 MachineMemOperand::MODereferenceable;
1967 *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
1968 DAG.getEVTAlignment(PtrTy));
1969 Node->setMemRefs(MemRefs, MemRefs + 1);
1970 }
1971 return SDValue(Node, 0);
1972}
1973
1974/// Codegen a new tail for a stack protector check ParentMBB which has had its
1975/// tail spliced into a stack protector check success bb.
1976///
1977/// For a high level explanation of how this fits into the stack protector
1978/// generation see the comment on the declaration of class
1979/// StackProtectorDescriptor.
1980void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
1981 MachineBasicBlock *ParentBB) {
1982
1983 // First create the loads to the guard/stack slot for the comparison.
1984 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1985 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
1986
1987 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
1988 int FI = MFI.getStackProtectorIndex();
1989
1990 SDValue Guard;
1991 SDLoc dl = getCurSDLoc();
1992 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
1993 const Module &M = *ParentBB->getParent()->getFunction()->getParent();
1994 unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
1995
1996 // Generate code to load the content of the guard slot.
1997 SDValue StackSlot = DAG.getLoad(
1998 PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
1999 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2000 MachineMemOperand::MOVolatile);
2001
2002 // Retrieve guard check function, nullptr if instrumentation is inlined.
2003 if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2004 // The target provides a guard check function to validate the guard value.
2005 // Generate a call to that function with the content of the guard slot as
2006 // argument.
2007 auto *Fn = cast<Function>(GuardCheck);
2008 FunctionType *FnTy = Fn->getFunctionType();
2009 assert(FnTy->getNumParams() == 1 && "Invalid function signature")((FnTy->getNumParams() == 1 && "Invalid function signature"
) ? static_cast<void> (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2009, __PRETTY_FUNCTION__))
;
2010
2011 TargetLowering::ArgListTy Args;
2012 TargetLowering::ArgListEntry Entry;
2013 Entry.Node = StackSlot;
2014 Entry.Ty = FnTy->getParamType(0);
2015 if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2016 Entry.IsInReg = true;
2017 Args.push_back(Entry);
2018
2019 TargetLowering::CallLoweringInfo CLI(DAG);
2020 CLI.setDebugLoc(getCurSDLoc())
2021 .setChain(DAG.getEntryNode())
2022 .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2023 getValue(GuardCheck), std::move(Args));
2024
2025 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2026 DAG.setRoot(Result.second);
2027 return;
2028 }
2029
2030 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2031 // Otherwise, emit a volatile load to retrieve the stack guard value.
2032 SDValue Chain = DAG.getEntryNode();
2033 if (TLI.useLoadStackGuardNode()) {
2034 Guard = getLoadStackGuard(DAG, dl, Chain);
2035 } else {
2036 const Value *IRGuard = TLI.getSDagStackGuard(M);
2037 SDValue GuardPtr = getValue(IRGuard);
2038
2039 Guard =
2040 DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2041 Align, MachineMemOperand::MOVolatile);
2042 }
2043
2044 // Perform the comparison via a subtract/getsetcc.
2045 EVT VT = Guard.getValueType();
2046 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, StackSlot);
2047
2048 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2049 *DAG.getContext(),
2050 Sub.getValueType()),
2051 Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2052
2053 // If the sub is not 0, then we know the guard/stackslot do not equal, so
2054 // branch to failure MBB.
2055 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2056 MVT::Other, StackSlot.getOperand(0),
2057 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2058 // Otherwise branch to success MBB.
2059 SDValue Br = DAG.getNode(ISD::BR, dl,
2060 MVT::Other, BrCond,
2061 DAG.getBasicBlock(SPD.getSuccessMBB()));
2062
2063 DAG.setRoot(Br);
2064}
2065
2066/// Codegen the failure basic block for a stack protector check.
2067///
2068/// A failure stack protector machine basic block consists simply of a call to
2069/// __stack_chk_fail().
2070///
2071/// For a high level explanation of how this fits into the stack protector
2072/// generation see the comment on the declaration of class
2073/// StackProtectorDescriptor.
2074void
2075SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2076 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2077 SDValue Chain =
2078 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2079 None, false, getCurSDLoc(), false, false).second;
2080 DAG.setRoot(Chain);
2081}
2082
2083/// visitBitTestHeader - This function emits necessary code to produce value
2084/// suitable for "bit tests"
2085void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2086 MachineBasicBlock *SwitchBB) {
2087 SDLoc dl = getCurSDLoc();
2088
2089 // Subtract the minimum value
2090 SDValue SwitchOp = getValue(B.SValue);
2091 EVT VT = SwitchOp.getValueType();
2092 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2093 DAG.getConstant(B.First, dl, VT));
2094
2095 // Check range
2096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2097 SDValue RangeCmp = DAG.getSetCC(
2098 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2099 Sub.getValueType()),
2100 Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2101
2102 // Determine the type of the test operands.
2103 bool UsePtrType = false;
2104 if (!TLI.isTypeLegal(VT))
2105 UsePtrType = true;
2106 else {
2107 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2108 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2109 // Switch table case range are encoded into series of masks.
2110 // Just use pointer type, it's guaranteed to fit.
2111 UsePtrType = true;
2112 break;
2113 }
2114 }
2115 if (UsePtrType) {
2116 VT = TLI.getPointerTy(DAG.getDataLayout());
2117 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2118 }
2119
2120 B.RegVT = VT.getSimpleVT();
2121 B.Reg = FuncInfo.CreateReg(B.RegVT);
2122 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2123
2124 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2125
2126 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2127 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2128 SwitchBB->normalizeSuccProbs();
2129
2130 SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2131 MVT::Other, CopyTo, RangeCmp,
2132 DAG.getBasicBlock(B.Default));
2133
2134 // Avoid emitting unnecessary branches to the next block.
2135 if (MBB != NextBlock(SwitchBB))
2136 BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2137 DAG.getBasicBlock(MBB));
2138
2139 DAG.setRoot(BrRange);
2140}
2141
2142/// visitBitTestCase - this function produces one "bit test"
2143void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2144 MachineBasicBlock* NextMBB,
2145 BranchProbability BranchProbToNext,
2146 unsigned Reg,
2147 BitTestCase &B,
2148 MachineBasicBlock *SwitchBB) {
2149 SDLoc dl = getCurSDLoc();
2150 MVT VT = BB.RegVT;
2151 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2152 SDValue Cmp;
2153 unsigned PopCount = countPopulation(B.Mask);
2154 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2155 if (PopCount == 1) {
2156 // Testing for a single bit; just compare the shift count with what it
2157 // would need to be to shift a 1 bit in that position.
2158 Cmp = DAG.getSetCC(
2159 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2160 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2161 ISD::SETEQ);
2162 } else if (PopCount == BB.Range) {
2163 // There is only one zero bit in the range, test for it directly.
2164 Cmp = DAG.getSetCC(
2165 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2166 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2167 ISD::SETNE);
2168 } else {
2169 // Make desired shift
2170 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2171 DAG.getConstant(1, dl, VT), ShiftOp);
2172
2173 // Emit bit tests and jumps
2174 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2175 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2176 Cmp = DAG.getSetCC(
2177 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2178 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2179 }
2180
2181 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2182 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2183 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2184 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2185 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2186 // one as they are relative probabilities (and thus work more like weights),
2187 // and hence we need to normalize them to let the sum of them become one.
2188 SwitchBB->normalizeSuccProbs();
2189
2190 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2191 MVT::Other, getControlRoot(),
2192 Cmp, DAG.getBasicBlock(B.TargetBB));
2193
2194 // Avoid emitting unnecessary branches to the next block.
2195 if (NextMBB != NextBlock(SwitchBB))
2196 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2197 DAG.getBasicBlock(NextMBB));
2198
2199 DAG.setRoot(BrAnd);
2200}
2201
2202void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2203 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2204
2205 // Retrieve successors. Look through artificial IR level blocks like
2206 // catchswitch for successors.
2207 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2208 const BasicBlock *EHPadBB = I.getSuccessor(1);
2209
2210 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2211 // have to do anything here to lower funclet bundles.
2212 assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2214, __PRETTY_FUNCTION__))
2213 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2214, __PRETTY_FUNCTION__))
2214 "Cannot lower invokes with arbitrary operand bundles yet!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2214, __PRETTY_FUNCTION__))
;
2215
2216 const Value *Callee(I.getCalledValue());
2217 const Function *Fn = dyn_cast<Function>(Callee);
2218 if (isa<InlineAsm>(Callee))
2219 visitInlineAsm(&I);
2220 else if (Fn && Fn->isIntrinsic()) {
2221 switch (Fn->getIntrinsicID()) {
2222 default:
2223 llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2223)
;
2224 case Intrinsic::donothing:
2225 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2226 break;
2227 case Intrinsic::experimental_patchpoint_void:
2228 case Intrinsic::experimental_patchpoint_i64:
2229 visitPatchpoint(&I, EHPadBB);
2230 break;
2231 case Intrinsic::experimental_gc_statepoint:
2232 LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2233 break;
2234 }
2235 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2236 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2237 // Eventually we will support lowering the @llvm.experimental.deoptimize
2238 // intrinsic, and right now there are no plans to support other intrinsics
2239 // with deopt state.
2240 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2241 } else {
2242 LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2243 }
2244
2245 // If the value of the invoke is used outside of its defining block, make it
2246 // available as a virtual register.
2247 // We already took care of the exported value for the statepoint instruction
2248 // during call to the LowerStatepoint.
2249 if (!isStatepoint(I)) {
2250 CopyToExportRegsIfNeeded(&I);
2251 }
2252
2253 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2254 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2255 BranchProbability EHPadBBProb =
2256 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2257 : BranchProbability::getZero();
2258 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2259
2260 // Update successor info.
2261 addSuccessorWithProb(InvokeMBB, Return);
2262 for (auto &UnwindDest : UnwindDests) {
2263 UnwindDest.first->setIsEHPad();
2264 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2265 }
2266 InvokeMBB->normalizeSuccProbs();
2267
2268 // Drop into normal successor.
2269 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2270 MVT::Other, getControlRoot(),
2271 DAG.getBasicBlock(Return)));
2272}
2273
2274void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2275 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2275)
;
2276}
2277
2278void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2279 assert(FuncInfo.MBB->isEHPad() &&((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2280, __PRETTY_FUNCTION__))
2280 "Call to landingpad not in landing pad!")((FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2280, __PRETTY_FUNCTION__))
;
2281
2282 MachineBasicBlock *MBB = FuncInfo.MBB;
2283 addLandingPadInfo(LP, *MBB);
2284
2285 // If there aren't registers to copy the values into (e.g., during SjLj
2286 // exceptions), then don't bother to create these DAG nodes.
2287 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2288 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2289 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2290 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2291 return;
2292
2293 // If landingpad's return type is token type, we don't create DAG nodes
2294 // for its exception pointer and selector value. The extraction of exception
2295 // pointer or selector value from token type landingpads is not currently
2296 // supported.
2297 if (LP.getType()->isTokenTy())
2298 return;
2299
2300 SmallVector<EVT, 2> ValueVTs;
2301 SDLoc dl = getCurSDLoc();
2302 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2303 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")((ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2303, __PRETTY_FUNCTION__))
;
2304
2305 // Get the two live-in registers as SDValues. The physregs have already been
2306 // copied into virtual registers.
2307 SDValue Ops[2];
2308 if (FuncInfo.ExceptionPointerVirtReg) {
2309 Ops[0] = DAG.getZExtOrTrunc(
2310 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2311 FuncInfo.ExceptionPointerVirtReg,
2312 TLI.getPointerTy(DAG.getDataLayout())),
2313 dl, ValueVTs[0]);
2314 } else {
2315 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2316 }
2317 Ops[1] = DAG.getZExtOrTrunc(
2318 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2319 FuncInfo.ExceptionSelectorVirtReg,
2320 TLI.getPointerTy(DAG.getDataLayout())),
2321 dl, ValueVTs[1]);
2322
2323 // Merge into one.
2324 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2325 DAG.getVTList(ValueVTs), Ops);
2326 setValue(&LP, Res);
2327}
2328
2329void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2330#ifndef NDEBUG
2331 for (const CaseCluster &CC : Clusters)
2332 assert(CC.Low == CC.High && "Input clusters must be single-case")((CC.Low == CC.High && "Input clusters must be single-case"
) ? static_cast<void> (0) : __assert_fail ("CC.Low == CC.High && \"Input clusters must be single-case\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2332, __PRETTY_FUNCTION__))
;
2333#endif
2334
2335 std::sort(Clusters.begin(), Clusters.end(),
2336 [](const CaseCluster &a, const CaseCluster &b) {
2337 return a.Low->getValue().slt(b.Low->getValue());
2338 });
2339
2340 // Merge adjacent clusters with the same destination.
2341 const unsigned N = Clusters.size();
2342 unsigned DstIndex = 0;
2343 for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2344 CaseCluster &CC = Clusters[SrcIndex];
2345 const ConstantInt *CaseVal = CC.Low;
2346 MachineBasicBlock *Succ = CC.MBB;
2347
2348 if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2349 (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2350 // If this case has the same successor and is a neighbour, merge it into
2351 // the previous cluster.
2352 Clusters[DstIndex - 1].High = CaseVal;
2353 Clusters[DstIndex - 1].Prob += CC.Prob;
2354 } else {
2355 std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2356 sizeof(Clusters[SrcIndex]));
2357 }
2358 }
2359 Clusters.resize(DstIndex);
2360}
2361
2362void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2363 MachineBasicBlock *Last) {
2364 // Update JTCases.
2365 for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2366 if (JTCases[i].first.HeaderBB == First)
2367 JTCases[i].first.HeaderBB = Last;
2368
2369 // Update BitTestCases.
2370 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2371 if (BitTestCases[i].Parent == First)
2372 BitTestCases[i].Parent = Last;
2373}
2374
2375void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2376 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2377
2378 // Update machine-CFG edges with unique successors.
2379 SmallSet<BasicBlock*, 32> Done;
2380 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2381 BasicBlock *BB = I.getSuccessor(i);
2382 bool Inserted = Done.insert(BB).second;
2383 if (!Inserted)
2384 continue;
2385
2386 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2387 addSuccessorWithProb(IndirectBrMBB, Succ);
2388 }
2389 IndirectBrMBB->normalizeSuccProbs();
2390
2391 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2392 MVT::Other, getControlRoot(),
2393 getValue(I.getAddress())));
2394}
2395
2396void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2397 if (DAG.getTarget().Options.TrapUnreachable)
2398 DAG.setRoot(
2399 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2400}
2401
2402void SelectionDAGBuilder::visitFSub(const User &I) {
2403 // -0.0 - X --> fneg
2404 Type *Ty = I.getType();
2405 if (isa<Constant>(I.getOperand(0)) &&
2406 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2407 SDValue Op2 = getValue(I.getOperand(1));
2408 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2409 Op2.getValueType(), Op2));
2410 return;
2411 }
2412
2413 visitBinary(I, ISD::FSUB);
2414}
2415
2416/// Checks if the given instruction performs a vector reduction, in which case
2417/// we have the freedom to alter the elements in the result as long as the
2418/// reduction of them stays unchanged.
2419static bool isVectorReductionOp(const User *I) {
2420 const Instruction *Inst = dyn_cast<Instruction>(I);
2421 if (!Inst || !Inst->getType()->isVectorTy())
2422 return false;
2423
2424 auto OpCode = Inst->getOpcode();
2425 switch (OpCode) {
2426 case Instruction::Add:
2427 case Instruction::Mul:
2428 case Instruction::And:
2429 case Instruction::Or:
2430 case Instruction::Xor:
2431 break;
2432 case Instruction::FAdd:
2433 case Instruction::FMul:
2434 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2435 if (FPOp->getFastMathFlags().unsafeAlgebra())
2436 break;
2437 LLVM_FALLTHROUGH[[clang::fallthrough]];
2438 default:
2439 return false;
2440 }
2441
2442 unsigned ElemNum = Inst->getType()->getVectorNumElements();
2443 unsigned ElemNumToReduce = ElemNum;
2444
2445 // Do DFS search on the def-use chain from the given instruction. We only
2446 // allow four kinds of operations during the search until we reach the
2447 // instruction that extracts the first element from the vector:
2448 //
2449 // 1. The reduction operation of the same opcode as the given instruction.
2450 //
2451 // 2. PHI node.
2452 //
2453 // 3. ShuffleVector instruction together with a reduction operation that
2454 // does a partial reduction.
2455 //
2456 // 4. ExtractElement that extracts the first element from the vector, and we
2457 // stop searching the def-use chain here.
2458 //
2459 // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2460 // from 1-3 to the stack to continue the DFS. The given instruction is not
2461 // a reduction operation if we meet any other instructions other than those
2462 // listed above.
2463
2464 SmallVector<const User *, 16> UsersToVisit{Inst};
2465 SmallPtrSet<const User *, 16> Visited;
2466 bool ReduxExtracted = false;
2467
2468 while (!UsersToVisit.empty()) {
2469 auto User = UsersToVisit.back();
2470 UsersToVisit.pop_back();
2471 if (!Visited.insert(User).second)
2472 continue;
2473
2474 for (const auto &U : User->users()) {
2475 auto Inst = dyn_cast<Instruction>(U);
2476 if (!Inst)
2477 return false;
2478
2479 if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2480 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2481 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().unsafeAlgebra())
2482 return false;
2483 UsersToVisit.push_back(U);
2484 } else if (const ShuffleVectorInst *ShufInst =
2485 dyn_cast<ShuffleVectorInst>(U)) {
2486 // Detect the following pattern: A ShuffleVector instruction together
2487 // with a reduction that do partial reduction on the first and second
2488 // ElemNumToReduce / 2 elements, and store the result in
2489 // ElemNumToReduce / 2 elements in another vector.
2490
2491 unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2492 if (ResultElements < ElemNum)
2493 return false;
2494
2495 if (ElemNumToReduce == 1)
2496 return false;
2497 if (!isa<UndefValue>(U->getOperand(1)))
2498 return false;
2499 for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2500 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2501 return false;
2502 for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2503 if (ShufInst->getMaskValue(i) != -1)
2504 return false;
2505
2506 // There is only one user of this ShuffleVector instruction, which
2507 // must be a reduction operation.
2508 if (!U->hasOneUse())
2509 return false;
2510
2511 auto U2 = dyn_cast<Instruction>(*U->user_begin());
2512 if (!U2 || U2->getOpcode() != OpCode)
2513 return false;
2514
2515 // Check operands of the reduction operation.
2516 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2517 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2518 UsersToVisit.push_back(U2);
2519 ElemNumToReduce /= 2;
2520 } else
2521 return false;
2522 } else if (isa<ExtractElementInst>(U)) {
2523 // At this moment we should have reduced all elements in the vector.
2524 if (ElemNumToReduce != 1)
2525 return false;
2526
2527 const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2528 if (!Val || Val->getZExtValue() != 0)
2529 return false;
2530
2531 ReduxExtracted = true;
2532 } else
2533 return false;
2534 }
2535 }
2536 return ReduxExtracted;
2537}
2538
2539void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) {
2540 SDValue Op1 = getValue(I.getOperand(0));
2541 SDValue Op2 = getValue(I.getOperand(1));
2542
2543 bool nuw = false;
2544 bool nsw = false;
2545 bool exact = false;
2546 bool vec_redux = false;
2547 FastMathFlags FMF;
2548
2549 if (const OverflowingBinaryOperator *OFBinOp =
2550 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2551 nuw = OFBinOp->hasNoUnsignedWrap();
2552 nsw = OFBinOp->hasNoSignedWrap();
2553 }
2554 if (const PossiblyExactOperator *ExactOp =
2555 dyn_cast<const PossiblyExactOperator>(&I))
2556 exact = ExactOp->isExact();
2557 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(&I))
2558 FMF = FPOp->getFastMathFlags();
2559
2560 if (isVectorReductionOp(&I)) {
2561 vec_redux = true;
2562 DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Detected a reduction operation:"
<< I << "\n"; } } while (false)
;
2563 }
2564
2565 SDNodeFlags Flags;
2566 Flags.setExact(exact);
2567 Flags.setNoSignedWrap(nsw);
2568 Flags.setNoUnsignedWrap(nuw);
2569 Flags.setVectorReduction(vec_redux);
2570 Flags.setAllowReciprocal(FMF.allowReciprocal());
2571 Flags.setAllowContract(FMF.allowContract());
2572 Flags.setNoInfs(FMF.noInfs());
2573 Flags.setNoNaNs(FMF.noNaNs());
2574 Flags.setNoSignedZeros(FMF.noSignedZeros());
2575 Flags.setUnsafeAlgebra(FMF.unsafeAlgebra());
2576
2577 SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(),
2578 Op1, Op2, Flags);
2579 setValue(&I, BinNodeValue);
2580}
2581
2582void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2583 SDValue Op1 = getValue(I.getOperand(0));
2584 SDValue Op2 = getValue(I.getOperand(1));
2585
2586 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2587 Op2.getValueType(), DAG.getDataLayout());
2588
2589 // Coerce the shift amount to the right type if we can.
2590 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2591 unsigned ShiftSize = ShiftTy.getSizeInBits();
2592 unsigned Op2Size = Op2.getValueSizeInBits();
2593 SDLoc DL = getCurSDLoc();
2594
2595 // If the operand is smaller than the shift count type, promote it.
2596 if (ShiftSize > Op2Size)
2597 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2598
2599 // If the operand is larger than the shift count type but the shift
2600 // count type has enough bits to represent any shift value, truncate
2601 // it now. This is a common case and it exposes the truncate to
2602 // optimization early.
2603 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2604 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2605 // Otherwise we'll need to temporarily settle for some other convenient
2606 // type. Type legalization will make adjustments once the shiftee is split.
2607 else
2608 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2609 }
2610
2611 bool nuw = false;
2612 bool nsw = false;
2613 bool exact = false;
2614
2615 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2616
2617 if (const OverflowingBinaryOperator *OFBinOp =
2618 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2619 nuw = OFBinOp->hasNoUnsignedWrap();
2620 nsw = OFBinOp->hasNoSignedWrap();
2621 }
2622 if (const PossiblyExactOperator *ExactOp =
2623 dyn_cast<const PossiblyExactOperator>(&I))
2624 exact = ExactOp->isExact();
2625 }
2626 SDNodeFlags Flags;
2627 Flags.setExact(exact);
2628 Flags.setNoSignedWrap(nsw);
2629 Flags.setNoUnsignedWrap(nuw);
2630 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2631 Flags);
2632 setValue(&I, Res);
2633}
2634
2635void SelectionDAGBuilder::visitSDiv(const User &I) {
2636 SDValue Op1 = getValue(I.getOperand(0));
2637 SDValue Op2 = getValue(I.getOperand(1));
2638
2639 SDNodeFlags Flags;
2640 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2641 cast<PossiblyExactOperator>(&I)->isExact());
2642 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2643 Op2, Flags));
2644}
2645
2646void SelectionDAGBuilder::visitICmp(const User &I) {
2647 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2648 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2649 predicate = IC->getPredicate();
2650 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2651 predicate = ICmpInst::Predicate(IC->getPredicate());
2652 SDValue Op1 = getValue(I.getOperand(0));
2653 SDValue Op2 = getValue(I.getOperand(1));
2654 ISD::CondCode Opcode = getICmpCondCode(predicate);
2655
2656 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2657 I.getType());
2658 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2659}
2660
2661void SelectionDAGBuilder::visitFCmp(const User &I) {
2662 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2663 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2664 predicate = FC->getPredicate();
2665 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2666 predicate = FCmpInst::Predicate(FC->getPredicate());
2667 SDValue Op1 = getValue(I.getOperand(0));
2668 SDValue Op2 = getValue(I.getOperand(1));
2669 ISD::CondCode Condition = getFCmpCondCode(predicate);
2670
2671 // FIXME: Fcmp instructions have fast-math-flags in IR, so we should use them.
2672 // FIXME: We should propagate the fast-math-flags to the DAG node itself for
2673 // further optimization, but currently FMF is only applicable to binary nodes.
2674 if (TM.Options.NoNaNsFPMath)
2675 Condition = getFCmpCodeWithoutNaN(Condition);
2676 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2677 I.getType());
2678 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2679}
2680
2681// Check if the condition of the select has one use or two users that are both
2682// selects with the same condition.
2683static bool hasOnlySelectUsers(const Value *Cond) {
2684 return all_of(Cond->users(), [](const Value *V) {
2685 return isa<SelectInst>(V);
2686 });
2687}
2688
2689void SelectionDAGBuilder::visitSelect(const User &I) {
2690 SmallVector<EVT, 4> ValueVTs;
2691 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2692 ValueVTs);
2693 unsigned NumValues = ValueVTs.size();
2694 if (NumValues == 0) return;
2695
2696 SmallVector<SDValue, 4> Values(NumValues);
2697 SDValue Cond = getValue(I.getOperand(0));
2698 SDValue LHSVal = getValue(I.getOperand(1));
2699 SDValue RHSVal = getValue(I.getOperand(2));
2700 auto BaseOps = {Cond};
2701 ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2702 ISD::VSELECT : ISD::SELECT;
2703
2704 // Min/max matching is only viable if all output VTs are the same.
2705 if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2706 EVT VT = ValueVTs[0];
2707 LLVMContext &Ctx = *DAG.getContext();
2708 auto &TLI = DAG.getTargetLoweringInfo();
2709
2710 // We care about the legality of the operation after it has been type
2711 // legalized.
2712 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2713 VT != TLI.getTypeToTransformTo(Ctx, VT))
2714 VT = TLI.getTypeToTransformTo(Ctx, VT);
2715
2716 // If the vselect is legal, assume we want to leave this as a vector setcc +
2717 // vselect. Otherwise, if this is going to be scalarized, we want to see if
2718 // min/max is legal on the scalar type.
2719 bool UseScalarMinMax = VT.isVector() &&
2720 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2721
2722 Value *LHS, *RHS;
2723 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2724 ISD::NodeType Opc = ISD::DELETED_NODE;
2725 switch (SPR.Flavor) {
2726 case SPF_UMAX: Opc = ISD::UMAX; break;
2727 case SPF_UMIN: Opc = ISD::UMIN; break;
2728 case SPF_SMAX: Opc = ISD::SMAX; break;
2729 case SPF_SMIN: Opc = ISD::SMIN; break;
2730 case SPF_FMINNUM:
2731 switch (SPR.NaNBehavior) {
2732 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2732)
;
2733 case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2734 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2735 case SPNB_RETURNS_ANY: {
2736 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2737 Opc = ISD::FMINNUM;
2738 else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2739 Opc = ISD::FMINNAN;
2740 else if (UseScalarMinMax)
2741 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2742 ISD::FMINNUM : ISD::FMINNAN;
2743 break;
2744 }
2745 }
2746 break;
2747 case SPF_FMAXNUM:
2748 switch (SPR.NaNBehavior) {
2749 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2749)
;
2750 case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2751 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2752 case SPNB_RETURNS_ANY:
2753
2754 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2755 Opc = ISD::FMAXNUM;
2756 else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2757 Opc = ISD::FMAXNAN;
2758 else if (UseScalarMinMax)
2759 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2760 ISD::FMAXNUM : ISD::FMAXNAN;
2761 break;
2762 }
2763 break;
2764 default: break;
2765 }
2766
2767 if (Opc != ISD::DELETED_NODE &&
2768 (TLI.isOperationLegalOrCustom(Opc, VT) ||
2769 (UseScalarMinMax &&
2770 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
2771 // If the underlying comparison instruction is used by any other
2772 // instruction, the consumed instructions won't be destroyed, so it is
2773 // not profitable to convert to a min/max.
2774 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
2775 OpCode = Opc;
2776 LHSVal = getValue(LHS);
2777 RHSVal = getValue(RHS);
2778 BaseOps = {};
2779 }
2780 }
2781
2782 for (unsigned i = 0; i != NumValues; ++i) {
2783 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
2784 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
2785 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
2786 Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
2787 LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
2788 Ops);
2789 }
2790
2791 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
2792 DAG.getVTList(ValueVTs), Values));
2793}
2794
2795void SelectionDAGBuilder::visitTrunc(const User &I) {
2796 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
2797 SDValue N = getValue(I.getOperand(0));
2798 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2799 I.getType());
2800 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
2801}
2802
2803void SelectionDAGBuilder::visitZExt(const User &I) {
2804 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2805 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
2806 SDValue N = getValue(I.getOperand(0));
2807 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2808 I.getType());
2809 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
2810}
2811
2812void SelectionDAGBuilder::visitSExt(const User &I) {
2813 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
2814 // SExt also can't be a cast to bool for same reason. So, nothing much to do
2815 SDValue N = getValue(I.getOperand(0));
2816 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2817 I.getType());
2818 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
2819}
2820
2821void SelectionDAGBuilder::visitFPTrunc(const User &I) {
2822 // FPTrunc is never a no-op cast, no need to check
2823 SDValue N = getValue(I.getOperand(0));
2824 SDLoc dl = getCurSDLoc();
2825 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2826 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2827 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
2828 DAG.getTargetConstant(
2829 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
2830}
2831
2832void SelectionDAGBuilder::visitFPExt(const User &I) {
2833 // FPExt is never a no-op cast, no need to check
2834 SDValue N = getValue(I.getOperand(0));
2835 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2836 I.getType());
2837 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
2838}
2839
2840void SelectionDAGBuilder::visitFPToUI(const User &I) {
2841 // FPToUI is never a no-op cast, no need to check
2842 SDValue N = getValue(I.getOperand(0));
2843 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2844 I.getType());
2845 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
2846}
2847
2848void SelectionDAGBuilder::visitFPToSI(const User &I) {
2849 // FPToSI is never a no-op cast, no need to check
2850 SDValue N = getValue(I.getOperand(0));
2851 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2852 I.getType());
2853 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
2854}
2855
2856void SelectionDAGBuilder::visitUIToFP(const User &I) {
2857 // UIToFP is never a no-op cast, no need to check
2858 SDValue N = getValue(I.getOperand(0));
2859 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2860 I.getType());
2861 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
2862}
2863
2864void SelectionDAGBuilder::visitSIToFP(const User &I) {
2865 // SIToFP is never a no-op cast, no need to check
2866 SDValue N = getValue(I.getOperand(0));
2867 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2868 I.getType());
2869 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
2870}
2871
2872void SelectionDAGBuilder::visitPtrToInt(const User &I) {
2873 // What to do depends on the size of the integer and the size of the pointer.
2874 // We can either truncate, zero extend, or no-op, accordingly.
2875 SDValue N = getValue(I.getOperand(0));
2876 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2877 I.getType());
2878 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2879}
2880
2881void SelectionDAGBuilder::visitIntToPtr(const User &I) {
2882 // What to do depends on the size of the integer and the size of the pointer.
2883 // We can either truncate, zero extend, or no-op, accordingly.
2884 SDValue N = getValue(I.getOperand(0));
2885 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2886 I.getType());
2887 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
2888}
2889
2890void SelectionDAGBuilder::visitBitCast(const User &I) {
2891 SDValue N = getValue(I.getOperand(0));
2892 SDLoc dl = getCurSDLoc();
2893 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2894 I.getType());
2895
2896 // BitCast assures us that source and destination are the same size so this is
2897 // either a BITCAST or a no-op.
2898 if (DestVT != N.getValueType())
2899 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
2900 DestVT, N)); // convert types.
2901 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
2902 // might fold any kind of constant expression to an integer constant and that
2903 // is not what we are looking for. Only recognize a bitcast of a genuine
2904 // constant integer as an opaque constant.
2905 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
2906 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
2907 /*isOpaque*/true));
2908 else
2909 setValue(&I, N); // noop cast.
2910}
2911
2912void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
2913 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2914 const Value *SV = I.getOperand(0);
2915 SDValue N = getValue(SV);
2916 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2917
2918 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
2919 unsigned DestAS = I.getType()->getPointerAddressSpace();
2920
2921 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
2922 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
2923
2924 setValue(&I, N);
2925}
2926
2927void SelectionDAGBuilder::visitInsertElement(const User &I) {
2928 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2929 SDValue InVec = getValue(I.getOperand(0));
2930 SDValue InVal = getValue(I.getOperand(1));
2931 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
2932 TLI.getVectorIdxTy(DAG.getDataLayout()));
2933 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
2934 TLI.getValueType(DAG.getDataLayout(), I.getType()),
2935 InVec, InVal, InIdx));
2936}
2937
2938void SelectionDAGBuilder::visitExtractElement(const User &I) {
2939 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2940 SDValue InVec = getValue(I.getOperand(0));
2941 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
2942 TLI.getVectorIdxTy(DAG.getDataLayout()));
2943 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
2944 TLI.getValueType(DAG.getDataLayout(), I.getType()),
2945 InVec, InIdx));
2946}
2947
2948void SelectionDAGBuilder::visitShuffleVector(const User &I) {
2949 SDValue Src1 = getValue(I.getOperand(0));
2950 SDValue Src2 = getValue(I.getOperand(1));
2951 SDLoc DL = getCurSDLoc();
2952
2953 SmallVector<int, 8> Mask;
2954 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
2955 unsigned MaskNumElts = Mask.size();
2956
2957 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2958 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
2959 EVT SrcVT = Src1.getValueType();
2960 unsigned SrcNumElts = SrcVT.getVectorNumElements();
2961
2962 if (SrcNumElts == MaskNumElts) {
2963 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
2964 return;
2965 }
2966
2967 // Normalize the shuffle vector since mask and vector length don't match.
2968 if (SrcNumElts < MaskNumElts) {
2969 // Mask is longer than the source vectors. We can use concatenate vector to
2970 // make the mask and vectors lengths match.
2971
2972 if (MaskNumElts % SrcNumElts == 0) {
2973 // Mask length is a multiple of the source vector length.
2974 // Check if the shuffle is some kind of concatenation of the input
2975 // vectors.
2976 unsigned NumConcat = MaskNumElts / SrcNumElts;
2977 bool IsConcat = true;
2978 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
2979 for (unsigned i = 0; i != MaskNumElts; ++i) {
2980 int Idx = Mask[i];
2981 if (Idx < 0)
2982 continue;
2983 // Ensure the indices in each SrcVT sized piece are sequential and that
2984 // the same source is used for the whole piece.
2985 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
2986 (ConcatSrcs[i / SrcNumElts] >= 0 &&
2987 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
2988 IsConcat = false;
2989 break;
2990 }
2991 // Remember which source this index came from.
2992 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
2993 }
2994
2995 // The shuffle is concatenating multiple vectors together. Just emit
2996 // a CONCAT_VECTORS operation.
2997 if (IsConcat) {
2998 SmallVector<SDValue, 8> ConcatOps;
2999 for (auto Src : ConcatSrcs) {
3000 if (Src < 0)
3001 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3002 else if (Src == 0)
3003 ConcatOps.push_back(Src1);
3004 else
3005 ConcatOps.push_back(Src2);
3006 }
3007 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3008 return;
3009 }
3010 }
3011
3012 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3013 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3014 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3015 PaddedMaskNumElts);
3016
3017 // Pad both vectors with undefs to make them the same length as the mask.
3018 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3019
3020 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3021 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3022 MOps1[0] = Src1;
3023 MOps2[0] = Src2;
3024
3025 Src1 = Src1.isUndef()
3026 ? DAG.getUNDEF(PaddedVT)
3027 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3028 Src2 = Src2.isUndef()
3029 ? DAG.getUNDEF(PaddedVT)
3030 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3031
3032 // Readjust mask for new input vector length.
3033 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3034 for (unsigned i = 0; i != MaskNumElts; ++i) {
3035 int Idx = Mask[i];
3036 if (Idx >= (int)SrcNumElts)
3037 Idx -= SrcNumElts - PaddedMaskNumElts;
3038 MappedOps[i] = Idx;
3039 }
3040
3041 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3042
3043 // If the concatenated vector was padded, extract a subvector with the
3044 // correct number of elements.
3045 if (MaskNumElts != PaddedMaskNumElts)
3046 Result = DAG.getNode(
3047 ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3048 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3049
3050 setValue(&I, Result);
3051 return;
3052 }
3053
3054 if (SrcNumElts > MaskNumElts) {
3055 // Analyze the access pattern of the vector to see if we can extract
3056 // two subvectors and do the shuffle.
3057 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3058 bool CanExtract = true;
3059 for (int Idx : Mask) {
3060 unsigned Input = 0;
3061 if (Idx < 0)
3062 continue;
3063
3064 if (Idx >= (int)SrcNumElts) {
3065 Input = 1;
3066 Idx -= SrcNumElts;
3067 }
3068
3069 // If all the indices come from the same MaskNumElts sized portion of
3070 // the sources we can use extract. Also make sure the extract wouldn't
3071 // extract past the end of the source.
3072 int NewStartIdx = alignDown(Idx, MaskNumElts);
3073 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3074 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3075 CanExtract = false;
3076 // Make sure we always update StartIdx as we use it to track if all
3077 // elements are undef.
3078 StartIdx[Input] = NewStartIdx;
3079 }
3080
3081 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3082 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3083 return;
3084 }
3085 if (CanExtract) {
3086 // Extract appropriate subvector and generate a vector shuffle
3087 for (unsigned Input = 0; Input < 2; ++Input) {
3088 SDValue &Src = Input == 0 ? Src1 : Src2;
3089 if (StartIdx[Input] < 0)
3090 Src = DAG.getUNDEF(VT);
3091 else {
3092 Src = DAG.getNode(
3093 ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3094 DAG.getConstant(StartIdx[Input], DL,
3095 TLI.getVectorIdxTy(DAG.getDataLayout())));
3096 }
3097 }
3098
3099 // Calculate new mask.
3100 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3101 for (int &Idx : MappedOps) {
3102 if (Idx >= (int)SrcNumElts)
3103 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3104 else if (Idx >= 0)
3105 Idx -= StartIdx[0];
3106 }
3107
3108 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3109 return;
3110 }
3111 }
3112
3113 // We can't use either concat vectors or extract subvectors so fall back to
3114 // replacing the shuffle with extract and build vector.
3115 // to insert and build vector.
3116 EVT EltVT = VT.getVectorElementType();
3117 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3118 SmallVector<SDValue,8> Ops;
3119 for (int Idx : Mask) {
3120 SDValue Res;
3121
3122 if (Idx < 0) {
3123 Res = DAG.getUNDEF(EltVT);
3124 } else {
3125 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3126 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3127
3128 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3129 EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3130 }
3131
3132 Ops.push_back(Res);
3133 }
3134
3135 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3136}
3137
3138void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3139 const Value *Op0 = I.getOperand(0);
3140 const Value *Op1 = I.getOperand(1);
3141 Type *AggTy = I.getType();
3142 Type *ValTy = Op1->getType();
3143 bool IntoUndef = isa<UndefValue>(Op0);
3144 bool FromUndef = isa<UndefValue>(Op1);
3145
3146 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3147
3148 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3149 SmallVector<EVT, 4> AggValueVTs;
3150 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3151 SmallVector<EVT, 4> ValValueVTs;
3152 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3153
3154 unsigned NumAggValues = AggValueVTs.size();
3155 unsigned NumValValues = ValValueVTs.size();
3156 SmallVector<SDValue, 4> Values(NumAggValues);
3157
3158 // Ignore an insertvalue that produces an empty object
3159 if (!NumAggValues) {
3160 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3161 return;
3162 }
3163
3164 SDValue Agg = getValue(Op0);
3165 unsigned i = 0;
3166 // Copy the beginning value(s) from the original aggregate.
3167 for (; i != LinearIndex; ++i)
3168 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3169 SDValue(Agg.getNode(), Agg.getResNo() + i);
3170 // Copy values from the inserted value(s).
3171 if (NumValValues) {
3172 SDValue Val = getValue(Op1);
3173 for (; i != LinearIndex + NumValValues; ++i)
3174 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3175 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3176 }
3177 // Copy remaining value(s) from the original aggregate.
3178 for (; i != NumAggValues; ++i)
3179 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3180 SDValue(Agg.getNode(), Agg.getResNo() + i);
3181
3182 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3183 DAG.getVTList(AggValueVTs), Values));
3184}
3185
3186void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3187 const Value *Op0 = I.getOperand(0);
3188 Type *AggTy = Op0->getType();
3189 Type *ValTy = I.getType();
3190 bool OutOfUndef = isa<UndefValue>(Op0);
3191
3192 unsigned LinearIndex = ComputeLinearIndex(AggTy, I.getIndices());
3193
3194 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3195 SmallVector<EVT, 4> ValValueVTs;
3196 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3197
3198 unsigned NumValValues = ValValueVTs.size();
3199
3200 // Ignore a extractvalue that produces an empty object
3201 if (!NumValValues) {
3202 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3203 return;
3204 }
3205
3206 SmallVector<SDValue, 4> Values(NumValValues);
3207
3208 SDValue Agg = getValue(Op0);
3209 // Copy out the selected value(s).
3210 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3211 Values[i - LinearIndex] =
3212 OutOfUndef ?
3213 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3214 SDValue(Agg.getNode(), Agg.getResNo() + i);
3215
3216 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3217 DAG.getVTList(ValValueVTs), Values));
3218}
3219
3220void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3221 Value *Op0 = I.getOperand(0);
3222 // Note that the pointer operand may be a vector of pointers. Take the scalar
3223 // element which holds a pointer.
3224 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3225 SDValue N = getValue(Op0);
3226 SDLoc dl = getCurSDLoc();
3227
3228 // Normalize Vector GEP - all scalar operands should be converted to the
3229 // splat vector.
3230 unsigned VectorWidth = I.getType()->isVectorTy() ?
3231 cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3232
3233 if (VectorWidth && !N.getValueType().isVector()) {
3234 LLVMContext &Context = *DAG.getContext();
3235 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3236 N = DAG.getSplatBuildVector(VT, dl, N);
3237 }
3238
3239 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3240 GTI != E; ++GTI) {
3241 const Value *Idx = GTI.getOperand();
3242 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3243 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3244 if (Field) {
3245 // N = N + Offset
3246 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3247
3248 // In an inbounds GEP with an offset that is nonnegative even when
3249 // interpreted as signed, assume there is no unsigned overflow.
3250 SDNodeFlags Flags;
3251 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3252 Flags.setNoUnsignedWrap(true);
3253
3254 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3255 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3256 }
3257 } else {
3258 MVT PtrTy =
3259 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout(), AS);
3260 unsigned PtrSize = PtrTy.getSizeInBits();
3261 APInt ElementSize(PtrSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3262
3263 // If this is a scalar constant or a splat vector of constants,
3264 // handle it quickly.
3265 const auto *CI = dyn_cast<ConstantInt>(Idx);
3266 if (!CI && isa<ConstantDataVector>(Idx) &&
3267 cast<ConstantDataVector>(Idx)->getSplatValue())
3268 CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3269
3270 if (CI) {
3271 if (CI->isZero())
3272 continue;
3273 APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize);
3274 LLVMContext &Context = *DAG.getContext();
3275 SDValue OffsVal = VectorWidth ?
3276 DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, PtrTy, VectorWidth)) :
3277 DAG.getConstant(Offs, dl, PtrTy);
3278
3279 // In an inbouds GEP with an offset that is nonnegative even when
3280 // interpreted as signed, assume there is no unsigned overflow.
3281 SDNodeFlags Flags;
3282 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3283 Flags.setNoUnsignedWrap(true);
3284
3285 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3286 continue;
3287 }
3288
3289 // N = N + Idx * ElementSize;
3290 SDValue IdxN = getValue(Idx);
3291
3292 if (!IdxN.getValueType().isVector() && VectorWidth) {
3293 MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth);
3294 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3295 }
3296
3297 // If the index is smaller or larger than intptr_t, truncate or extend
3298 // it.
3299 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3300
3301 // If this is a multiply by a power of two, turn it into a shl
3302 // immediately. This is a very common case.
3303 if (ElementSize != 1) {
3304 if (ElementSize.isPowerOf2()) {
3305 unsigned Amt = ElementSize.logBase2();
3306 IdxN = DAG.getNode(ISD::SHL, dl,
3307 N.getValueType(), IdxN,
3308 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3309 } else {
3310 SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3311 IdxN = DAG.getNode(ISD::MUL, dl,
3312 N.getValueType(), IdxN, Scale);
3313 }
3314 }
3315
3316 N = DAG.getNode(ISD::ADD, dl,
3317 N.getValueType(), N, IdxN);
3318 }
3319 }
3320
3321 setValue(&I, N);
3322}
3323
3324void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3325 // If this is a fixed sized alloca in the entry block of the function,
3326 // allocate it statically on the stack.
3327 if (FuncInfo.StaticAllocaMap.count(&I))
3328 return; // getValue will auto-populate this.
3329
3330 SDLoc dl = getCurSDLoc();
3331 Type *Ty = I.getAllocatedType();
3332 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3333 auto &DL = DAG.getDataLayout();
3334 uint64_t TySize = DL.getTypeAllocSize(Ty);
3335 unsigned Align =
3336 std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3337
3338 SDValue AllocSize = getValue(I.getArraySize());
3339
3340 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
3341 if (AllocSize.getValueType() != IntPtr)
3342 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3343
3344 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3345 AllocSize,
3346 DAG.getConstant(TySize, dl, IntPtr));
3347
3348 // Handle alignment. If the requested alignment is less than or equal to
3349 // the stack alignment, ignore it. If the size is greater than or equal to
3350 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3351 unsigned StackAlign =
3352 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3353 if (Align <= StackAlign)
3354 Align = 0;
3355
3356 // Round the size of the allocation up to the stack alignment size
3357 // by add SA-1 to the size. This doesn't overflow because we're computing
3358 // an address inside an alloca.
3359 SDNodeFlags Flags;
3360 Flags.setNoUnsignedWrap(true);
3361 AllocSize = DAG.getNode(ISD::ADD, dl,
3362 AllocSize.getValueType(), AllocSize,
3363 DAG.getIntPtrConstant(StackAlign - 1, dl), Flags);
3364
3365 // Mask out the low bits for alignment purposes.
3366 AllocSize = DAG.getNode(ISD::AND, dl,
3367 AllocSize.getValueType(), AllocSize,
3368 DAG.getIntPtrConstant(~(uint64_t)(StackAlign - 1),
3369 dl));
3370
3371 SDValue Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align, dl) };
3372 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3373 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3374 setValue(&I, DSA);
3375 DAG.setRoot(DSA.getValue(1));
3376
3377 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())((FuncInfo.MF->getFrameInfo().hasVarSizedObjects()) ? static_cast
<void> (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3377, __PRETTY_FUNCTION__))
;
3378}
3379
3380void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3381 if (I.isAtomic())
3382 return visitAtomicLoad(I);
3383
3384 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3385 const Value *SV = I.getOperand(0);
3386 if (TLI.supportSwiftError()) {
3387 // Swifterror values can come from either a function parameter with
3388 // swifterror attribute or an alloca with swifterror attribute.
3389 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3390 if (Arg->hasSwiftErrorAttr())
3391 return visitLoadFromSwiftError(I);
3392 }
3393
3394 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3395 if (Alloca->isSwiftError())
3396 return visitLoadFromSwiftError(I);
3397 }
3398 }
3399
3400 SDValue Ptr = getValue(SV);
3401
3402 Type *Ty = I.getType();
3403
3404 bool isVolatile = I.isVolatile();
3405 bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3406 bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3407 bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3408 unsigned Alignment = I.getAlignment();
3409
3410 AAMDNodes AAInfo;
3411 I.getAAMetadata(AAInfo);
3412 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3413
3414 SmallVector<EVT, 4> ValueVTs;
3415 SmallVector<uint64_t, 4> Offsets;
3416 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3417 unsigned NumValues = ValueVTs.size();
3418 if (NumValues == 0)
3419 return;
3420
3421 SDValue Root;
3422 bool ConstantMemory = false;
3423 if (isVolatile || NumValues > MaxParallelChains)
3424 // Serialize volatile loads with other side effects.
3425 Root = getRoot();
3426 else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3427 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3428 // Do not serialize (non-volatile) loads of constant memory with anything.
3429 Root = DAG.getEntryNode();
3430 ConstantMemory = true;
3431 } else {
3432 // Do not serialize non-volatile loads against each other.
3433 Root = DAG.getRoot();
3434 }
3435
3436 SDLoc dl = getCurSDLoc();
3437
3438 if (isVolatile)
3439 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3440
3441 // An aggregate load cannot wrap around the address space, so offsets to its
3442 // parts don't wrap either.
3443 SDNodeFlags Flags;
3444 Flags.setNoUnsignedWrap(true);
3445
3446 SmallVector<SDValue, 4> Values(NumValues);
3447 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3448 EVT PtrVT = Ptr.getValueType();
3449 unsigned ChainI = 0;
3450 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3451 // Serializing loads here may result in excessive register pressure, and
3452 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3453 // could recover a bit by hoisting nodes upward in the chain by recognizing
3454 // they are side-effect free or do not alias. The optimizer should really
3455 // avoid this case by converting large object/array copies to llvm.memcpy
3456 // (MaxParallelChains should always remain as failsafe).
3457 if (ChainI == MaxParallelChains) {
3458 assert(PendingLoads.empty() && "PendingLoads must be serialized first")((PendingLoads.empty() && "PendingLoads must be serialized first"
) ? static_cast<void> (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3458, __PRETTY_FUNCTION__))
;
3459 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3460 makeArrayRef(Chains.data(), ChainI));
3461 Root = Chain;
3462 ChainI = 0;
3463 }
3464 SDValue A = DAG.getNode(ISD::ADD, dl,
3465 PtrVT, Ptr,
3466 DAG.getConstant(Offsets[i], dl, PtrVT),
3467 Flags);
3468 auto MMOFlags = MachineMemOperand::MONone;
3469 if (isVolatile)
3470 MMOFlags |= MachineMemOperand::MOVolatile;
3471 if (isNonTemporal)
3472 MMOFlags |= MachineMemOperand::MONonTemporal;
3473 if (isInvariant)
3474 MMOFlags |= MachineMemOperand::MOInvariant;
3475 if (isDereferenceable)
3476 MMOFlags |= MachineMemOperand::MODereferenceable;
3477
3478 SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3479 MachinePointerInfo(SV, Offsets[i]), Alignment,
3480 MMOFlags, AAInfo, Ranges);
3481
3482 Values[i] = L;
3483 Chains[ChainI] = L.getValue(1);
3484 }
3485
3486 if (!ConstantMemory) {
3487 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3488 makeArrayRef(Chains.data(), ChainI));
3489 if (isVolatile)
3490 DAG.setRoot(Chain);
3491 else
3492 PendingLoads.push_back(Chain);
3493 }
3494
3495 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3496 DAG.getVTList(ValueVTs), Values));
3497}
3498
3499void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3500 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3501 assert(TLI.supportSwiftError() &&((TLI.supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("TLI.supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3502, __PRETTY_FUNCTION__))
3502 "call visitStoreToSwiftError when backend supports swifterror")((TLI.supportSwiftError() && "call visitStoreToSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("TLI.supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3502, __PRETTY_FUNCTION__))
;
3503
3504 SmallVector<EVT, 4> ValueVTs;
3505 SmallVector<uint64_t, 4> Offsets;
3506 const Value *SrcV = I.getOperand(0);
3507 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3508 SrcV->getType(), ValueVTs, &Offsets);
3509 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3510, __PRETTY_FUNCTION__))
3510 "expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3510, __PRETTY_FUNCTION__))
;
3511
3512 SDValue Src = getValue(SrcV);
3513 // Create a virtual register, then update the virtual register.
3514 auto &DL = DAG.getDataLayout();
3515 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3516 unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
3517 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3518 // Chain can be getRoot or getControlRoot.
3519 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3520 SDValue(Src.getNode(), Src.getResNo()));
3521 DAG.setRoot(CopyNode);
3522 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3523}
3524
3525void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3526 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3527, __PRETTY_FUNCTION__))
3527 "call visitLoadFromSwiftError when backend supports swifterror")((DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? static_cast<void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3527, __PRETTY_FUNCTION__))
;
3528
3529 assert(!I.isVolatile() &&((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3532, __PRETTY_FUNCTION__))
3530 I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3532, __PRETTY_FUNCTION__))
3531 I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3532, __PRETTY_FUNCTION__))
3532 "Support volatile, non temporal, invariant for load_from_swift_error")((!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal
) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load
) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? static_cast<void> (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3532, __PRETTY_FUNCTION__))
;
3533
3534 const Value *SV = I.getOperand(0);
3535 Type *Ty = I.getType();
3536 AAMDNodes AAInfo;
3537 I.getAAMetadata(AAInfo);
3538 assert((!AA || !AA->pointsToConstantMemory(MemoryLocation((((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG
.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3540, __PRETTY_FUNCTION__))
3539 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG
.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3540, __PRETTY_FUNCTION__))
3540 "load_from_swift_error should not be constant memory")(((!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG
.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && "load_from_swift_error should not be constant memory"
) ? static_cast<void> (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3540, __PRETTY_FUNCTION__))
;
3541
3542 SmallVector<EVT, 4> ValueVTs;
3543 SmallVector<uint64_t, 4> Offsets;
3544 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3545 ValueVTs, &Offsets);
3546 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3547, __PRETTY_FUNCTION__))
3547 "expect a single EVT for swifterror")((ValueVTs.size() == 1 && Offsets[0] == 0 && "expect a single EVT for swifterror"
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3547, __PRETTY_FUNCTION__))
;
3548
3549 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3550 SDValue L = DAG.getCopyFromReg(
3551 getRoot(), getCurSDLoc(),
3552 FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, SV), ValueVTs[0]);
3553
3554 setValue(&I, L);
3555}
3556
3557void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3558 if (I.isAtomic())
3559 return visitAtomicStore(I);
3560
3561 const Value *SrcV = I.getOperand(0);
3562 const Value *PtrV = I.getOperand(1);
3563
3564 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3565 if (TLI.supportSwiftError()) {
3566 // Swifterror values can come from either a function parameter with
3567 // swifterror attribute or an alloca with swifterror attribute.
3568 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3569 if (Arg->hasSwiftErrorAttr())
3570 return visitStoreToSwiftError(I);
3571 }
3572
3573 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3574 if (Alloca->isSwiftError())
3575 return visitStoreToSwiftError(I);
3576 }
3577 }
3578
3579 SmallVector<EVT, 4> ValueVTs;
3580 SmallVector<uint64_t, 4> Offsets;
3581 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3582 SrcV->getType(), ValueVTs, &Offsets);
3583 unsigned NumValues = ValueVTs.size();
3584 if (NumValues == 0)
3585 return;
3586
3587 // Get the lowered operands. Note that we do this after
3588 // checking if NumResults is zero, because with zero results
3589 // the operands won't have values in the map.
3590 SDValue Src = getValue(SrcV);
3591 SDValue Ptr = getValue(PtrV);
3592
3593 SDValue Root = getRoot();
3594 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3595 SDLoc dl = getCurSDLoc();
3596 EVT PtrVT = Ptr.getValueType();
3597 unsigned Alignment = I.getAlignment();
3598 AAMDNodes AAInfo;
3599 I.getAAMetadata(AAInfo);
3600
3601 auto MMOFlags = MachineMemOperand::MONone;
3602 if (I.isVolatile())
3603 MMOFlags |= MachineMemOperand::MOVolatile;
3604 if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3605 MMOFlags |= MachineMemOperand::MONonTemporal;
3606
3607 // An aggregate load cannot wrap around the address space, so offsets to its
3608 // parts don't wrap either.
3609 SDNodeFlags Flags;
3610 Flags.setNoUnsignedWrap(true);
3611
3612 unsigned ChainI = 0;
3613 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3614 // See visitLoad comments.
3615 if (ChainI == MaxParallelChains) {
3616 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3617 makeArrayRef(Chains.data(), ChainI));
3618 Root = Chain;
3619 ChainI = 0;
3620 }
3621 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3622 DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3623 SDValue St = DAG.getStore(
3624 Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3625 MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3626 Chains[ChainI] = St;
3627 }
3628
3629 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3630 makeArrayRef(Chains.data(), ChainI));
3631 DAG.setRoot(StoreNode);
3632}
3633
3634void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3635 bool IsCompressing) {
3636 SDLoc sdl = getCurSDLoc();
3637
3638 auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3639 unsigned& Alignment) {
3640 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3641 Src0 = I.getArgOperand(0);
3642 Ptr = I.getArgOperand(1);
3643 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3644 Mask = I.getArgOperand(3);
3645 };
3646 auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3647 unsigned& Alignment) {
3648 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3649 Src0 = I.getArgOperand(0);
3650 Ptr = I.getArgOperand(1);
3651 Mask = I.getArgOperand(2);
3652 Alignment = 0;
3653 };
3654
3655 Value *PtrOperand, *MaskOperand, *Src0Operand;
3656 unsigned Alignment;
3657 if (IsCompressing)
3658 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3659 else
3660 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3661
3662 SDValue Ptr = getValue(PtrOperand);
3663 SDValue Src0 = getValue(Src0Operand);
3664 SDValue Mask = getValue(MaskOperand);
3665
3666 EVT VT = Src0.getValueType();
3667 if (!Alignment)
3668 Alignment = DAG.getEVTAlignment(VT);
3669
3670 AAMDNodes AAInfo;
3671 I.getAAMetadata(AAInfo);
3672
3673 MachineMemOperand *MMO =
3674 DAG.getMachineFunction().
3675 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3676 MachineMemOperand::MOStore, VT.getStoreSize(),
3677 Alignment, AAInfo);
3678 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3679 MMO, false /* Truncating */,
3680 IsCompressing);
3681 DAG.setRoot(StoreNode);
3682 setValue(&I, StoreNode);
3683}
3684
3685// Get a uniform base for the Gather/Scatter intrinsic.
3686// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3687// We try to represent it as a base pointer + vector of indices.
3688// Usually, the vector of pointers comes from a 'getelementptr' instruction.
3689// The first operand of the GEP may be a single pointer or a vector of pointers
3690// Example:
3691// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3692// or
3693// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3694// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3695//
3696// When the first GEP operand is a single pointer - it is the uniform base we
3697// are looking for. If first operand of the GEP is a splat vector - we
3698// extract the spalt value and use it as a uniform base.
3699// In all other cases the function returns 'false'.
3700//
3701static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3702 SelectionDAGBuilder* SDB) {
3703
3704 SelectionDAG& DAG = SDB->DAG;
3705 LLVMContext &Context = *DAG.getContext();
3706
3707 assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")((Ptr->getType()->isVectorTy() && "Uexpected pointer type"
) ? static_cast<void> (0) : __assert_fail ("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3707, __PRETTY_FUNCTION__))
;
3708 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3709 if (!GEP || GEP->getNumOperands() > 2)
3710 return false;
3711
3712 const Value *GEPPtr = GEP->getPointerOperand();
3713 if (!GEPPtr->getType()->isVectorTy())
3714 Ptr = GEPPtr;
3715 else if (!(Ptr = getSplatValue(GEPPtr)))
3716 return false;
3717
3718 Value *IndexVal = GEP->getOperand(1);
3719
3720 // The operands of the GEP may be defined in another basic block.
3721 // In this case we'll not find nodes for the operands.
3722 if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3723 return false;
3724
3725 Base = SDB->getValue(Ptr);
3726 Index = SDB->getValue(IndexVal);
3727
3728 // Suppress sign extension.
3729 if (SExtInst* Sext = dyn_cast<SExtInst>(IndexVal)) {
3730 if (SDB->findValue(Sext->getOperand(0))) {
3731 IndexVal = Sext->getOperand(0);
3732 Index = SDB->getValue(IndexVal);
3733 }
3734 }
3735 if (!Index.getValueType().isVector()) {
3736 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3737 EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3738 Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3739 }
3740 return true;
3741}
3742
3743void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3744 SDLoc sdl = getCurSDLoc();
3745
3746 // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3747 const Value *Ptr = I.getArgOperand(1);
3748 SDValue Src0 = getValue(I.getArgOperand(0));
3749 SDValue Mask = getValue(I.getArgOperand(3));
3750 EVT VT = Src0.getValueType();
3751 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3752 if (!Alignment)
3753 Alignment = DAG.getEVTAlignment(VT);
3754 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3755
3756 AAMDNodes AAInfo;
3757 I.getAAMetadata(AAInfo);
3758
3759 SDValue Base;
3760 SDValue Index;
3761 const Value *BasePtr = Ptr;
3762 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3763
3764 const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
3765 MachineMemOperand *MMO = DAG.getMachineFunction().
3766 getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
3767 MachineMemOperand::MOStore, VT.getStoreSize(),
3768 Alignment, AAInfo);
3769 if (!UniformBase) {
3770 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3771 Index = getValue(Ptr);
3772 }
3773 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index };
3774 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
3775 Ops, MMO);
3776 DAG.setRoot(Scatter);
3777 setValue(&I, Scatter);
3778}
3779
3780void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
3781 SDLoc sdl = getCurSDLoc();
3782
3783 auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3784 unsigned& Alignment) {
3785 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
3786 Ptr = I.getArgOperand(0);
3787 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
3788 Mask = I.getArgOperand(2);
3789 Src0 = I.getArgOperand(3);
3790 };
3791 auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3792 unsigned& Alignment) {
3793 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
3794 Ptr = I.getArgOperand(0);
3795 Alignment = 0;
3796 Mask = I.getArgOperand(1);
3797 Src0 = I.getArgOperand(2);
3798 };
3799
3800 Value *PtrOperand, *MaskOperand, *Src0Operand;
3801 unsigned Alignment;
3802 if (IsExpanding)
3803 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3804 else
3805 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3806
3807 SDValue Ptr = getValue(PtrOperand);
3808 SDValue Src0 = getValue(Src0Operand);
3809 SDValue Mask = getValue(MaskOperand);
3810
3811 EVT VT = Src0.getValueType();
3812 if (!Alignment)
3813 Alignment = DAG.getEVTAlignment(VT);
3814
3815 AAMDNodes AAInfo;
3816 I.getAAMetadata(AAInfo);
3817 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3818
3819 // Do not serialize masked loads of constant memory with anything.
3820 bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
3821 PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
3822 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
3823
3824 MachineMemOperand *MMO =
3825 DAG.getMachineFunction().
3826 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3827 MachineMemOperand::MOLoad, VT.getStoreSize(),
3828 Alignment, AAInfo, Ranges);
3829
3830 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
3831 ISD::NON_EXTLOAD, IsExpanding);
3832 if (AddToChain) {
3833 SDValue OutChain = Load.getValue(1);
3834 DAG.setRoot(OutChain);
3835 }
3836 setValue(&I, Load);
3837}
3838
3839void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
3840 SDLoc sdl = getCurSDLoc();
3841
3842 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
3843 const Value *Ptr = I.getArgOperand(0);
3844 SDValue Src0 = getValue(I.getArgOperand(3));
3845 SDValue Mask = getValue(I.getArgOperand(2));
3846
3847 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3848 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3849 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
3850 if (!Alignment)
3851 Alignment = DAG.getEVTAlignment(VT);
3852
3853 AAMDNodes AAInfo;
3854 I.getAAMetadata(AAInfo);
3855 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3856
3857 SDValue Root = DAG.getRoot();
3858 SDValue Base;
3859 SDValue Index;
3860 const Value *BasePtr = Ptr;
3861 bool UniformBase = getUniformBase(BasePtr, Base, Index, this);
3862 bool ConstantMemory = false;
3863 if (UniformBase &&
3864 AA && AA->pointsToConstantMemory(MemoryLocation(
3865 BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
3866 AAInfo))) {
3867 // Do not serialize (non-volatile) loads of constant memory with anything.
3868 Root = DAG.getEntryNode();
3869 ConstantMemory = true;
3870 }
3871
3872 MachineMemOperand *MMO =
3873 DAG.getMachineFunction().
3874 getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
3875 MachineMemOperand::MOLoad, VT.getStoreSize(),
3876 Alignment, AAInfo, Ranges);
3877
3878 if (!UniformBase) {
3879 Base = DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
3880 Index = getValue(Ptr);
3881 }
3882 SDValue Ops[] = { Root, Src0, Mask, Base, Index };
3883 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
3884 Ops, MMO);
3885
3886 SDValue OutChain = Gather.getValue(1);
3887 if (!ConstantMemory)
3888 PendingLoads.push_back(OutChain);
3889 setValue(&I, Gather);
3890}
3891
3892void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
3893 SDLoc dl = getCurSDLoc();
3894 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
3895 AtomicOrdering FailureOrder = I.getFailureOrdering();
3896 SynchronizationScope Scope = I.getSynchScope();
3897
3898 SDValue InChain = getRoot();
3899
3900 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
3901 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
3902 SDValue L = DAG.getAtomicCmpSwap(
3903 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
3904 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
3905 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
3906 /*Alignment=*/ 0, SuccessOrder, FailureOrder, Scope);
3907
3908 SDValue OutChain = L.getValue(2);
3909
3910 setValue(&I, L);
3911 DAG.setRoot(OutChain);
3912}
3913
3914void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
3915 SDLoc dl = getCurSDLoc();
3916 ISD::NodeType NT;
3917 switch (I.getOperation()) {
3918 default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3918)
;
3919 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
3920 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
3921 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
3922 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
3923 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
3924 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
3925 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
3926 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
3927 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
3928 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
3929 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
3930 }
3931 AtomicOrdering Order = I.getOrdering();
3932 SynchronizationScope Scope = I.getSynchScope();
3933
3934 SDValue InChain = getRoot();
3935
3936 SDValue L =
3937 DAG.getAtomic(NT, dl,
3938 getValue(I.getValOperand()).getSimpleValueType(),
3939 InChain,
3940 getValue(I.getPointerOperand()),
3941 getValue(I.getValOperand()),
3942 I.getPointerOperand(),
3943 /* Alignment=*/ 0, Order, Scope);
3944
3945 SDValue OutChain = L.getValue(1);
3946
3947 setValue(&I, L);
3948 DAG.setRoot(OutChain);
3949}
3950
3951void SelectionDAGBuilder::visitFence(const FenceInst &I) {
3952 SDLoc dl = getCurSDLoc();
3953 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3954 SDValue Ops[3];
3955 Ops[0] = getRoot();
3956 Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
3957 TLI.getFenceOperandTy(DAG.getDataLayout()));
3958 Ops[2] = DAG.getConstant(I.getSynchScope(), dl,
3959 TLI.getFenceOperandTy(DAG.getDataLayout()));
3960 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
3961}
3962
3963void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
3964 SDLoc dl = getCurSDLoc();
3965 AtomicOrdering Order = I.getOrdering();
3966 SynchronizationScope Scope = I.getSynchScope();
3967
3968 SDValue InChain = getRoot();
3969
3970 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3971 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3972
3973 if (I.getAlignment() < VT.getSizeInBits() / 8)
3974 report_fatal_error("Cannot generate unaligned atomic load");
3975
3976 MachineMemOperand *MMO =
3977 DAG.getMachineFunction().
3978 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
3979 MachineMemOperand::MOVolatile |
3980 MachineMemOperand::MOLoad,
3981 VT.getStoreSize(),
3982 I.getAlignment() ? I.getAlignment() :
3983 DAG.getEVTAlignment(VT),
3984 AAMDNodes(), nullptr, Scope, Order);
3985
3986 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
3987 SDValue L =
3988 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
3989 getValue(I.getPointerOperand()), MMO);
3990
3991 SDValue OutChain = L.getValue(1);
3992
3993 setValue(&I, L);
3994 DAG.setRoot(OutChain);
3995}
3996
3997void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
3998 SDLoc dl = getCurSDLoc();
3999
4000 AtomicOrdering Order = I.getOrdering();
4001 SynchronizationScope Scope = I.getSynchScope();
4002
4003 SDValue InChain = getRoot();
4004
4005 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4006 EVT VT =
4007 TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4008
4009 if (I.getAlignment() < VT.getSizeInBits() / 8)
4010 report_fatal_error("Cannot generate unaligned atomic store");
4011
4012 SDValue OutChain =
4013 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4014 InChain,
4015 getValue(I.getPointerOperand()),
4016 getValue(I.getValueOperand()),
4017 I.getPointerOperand(), I.getAlignment(),
4018 Order, Scope);
4019
4020 DAG.setRoot(OutChain);
4021}
4022
4023/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4024/// node.
4025void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4026 unsigned Intrinsic) {
4027 // Ignore the callsite's attributes. A specific call site may be marked with
4028 // readnone, but the lowering code will expect the chain based on the
4029 // definition.
4030 const Function *F = I.getCalledFunction();
4031 bool HasChain = !F->doesNotAccessMemory();
4032 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4033
4034 // Build the operand list.
4035 SmallVector<SDValue, 8> Ops;
4036 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4037 if (OnlyLoad) {
4038 // We don't need to serialize loads against other loads.
4039 Ops.push_back(DAG.getRoot());
4040 } else {
4041 Ops.push_back(getRoot());
4042 }
4043 }
4044
4045 // Info is set by getTgtMemInstrinsic
4046 TargetLowering::IntrinsicInfo Info;
4047 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4048 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, Intrinsic);
4049
4050 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4051 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4052 Info.opc == ISD::INTRINSIC_W_CHAIN)
4053 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4054 TLI.getPointerTy(DAG.getDataLayout())));
4055
4056 // Add all operands of the call to the operand list.
4057 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4058 SDValue Op = getValue(I.getArgOperand(i));
4059 Ops.push_back(Op);
4060 }
4061
4062 SmallVector<EVT, 4> ValueVTs;
4063 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4064
4065 if (HasChain)
4066 ValueVTs.push_back(MVT::Other);
4067
4068 SDVTList VTs = DAG.getVTList(ValueVTs);
4069
4070 // Create the node.
4071 SDValue Result;
4072 if (IsTgtIntrinsic) {
4073 // This is target intrinsic that touches memory
4074 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(),
4075 VTs, Ops, Info.memVT,
4076 MachinePointerInfo(Info.ptrVal, Info.offset),
4077 Info.align, Info.vol,
4078 Info.readMem, Info.writeMem, Info.size);
4079 } else if (!HasChain) {
4080 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4081 } else if (!I.getType()->isVoidTy()) {
4082 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4083 } else {
4084 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4085 }
4086
4087 if (HasChain) {
4088 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4089 if (OnlyLoad)
4090 PendingLoads.push_back(Chain);
4091 else
4092 DAG.setRoot(Chain);
4093 }
4094
4095 if (!I.getType()->isVoidTy()) {
4096 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4097 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4098 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4099 } else
4100 Result = lowerRangeToAssertZExt(DAG, I, Result);
4101
4102 setValue(&I, Result);
4103 }
4104}
4105
4106/// GetSignificand - Get the significand and build it into a floating-point
4107/// number with exponent of 1:
4108///
4109/// Op = (Op & 0x007fffff) | 0x3f800000;
4110///
4111/// where Op is the hexadecimal representation of floating point value.
4112static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4113 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4114 DAG.getConstant(0x007fffff, dl, MVT::i32));
4115 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4116 DAG.getConstant(0x3f800000, dl, MVT::i32));
4117 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4118}
4119
4120/// GetExponent - Get the exponent:
4121///
4122/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4123///
4124/// where Op is the hexadecimal representation of floating point value.
4125static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4126 const TargetLowering &TLI, const SDLoc &dl) {
4127 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4128 DAG.getConstant(0x7f800000, dl, MVT::i32));
4129 SDValue t1 = DAG.getNode(
4130 ISD::SRL, dl, MVT::i32, t0,
4131 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4132 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4133 DAG.getConstant(127, dl, MVT::i32));
4134 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4135}
4136
4137/// getF32Constant - Get 32-bit floating point constant.
4138static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4139 const SDLoc &dl) {
4140 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4141 MVT::f32);
4142}
4143
4144static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4145 SelectionDAG &DAG) {
4146 // TODO: What fast-math-flags should be set on the floating-point nodes?
4147
4148 // IntegerPartOfX = ((int32_t)(t0);
4149 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4150
4151 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4152 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4153 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4154
4155 // IntegerPartOfX <<= 23;
4156 IntegerPartOfX = DAG.getNode(
4157 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4158 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4159 DAG.getDataLayout())));
4160
4161 SDValue TwoToFractionalPartOfX;
4162 if (LimitFloatPrecision <= 6) {
4163 // For floating-point precision of 6:
4164 //
4165 // TwoToFractionalPartOfX =
4166 // 0.997535578f +
4167 // (0.735607626f + 0.252464424f * x) * x;
4168 //
4169 // error 0.0144103317, which is 6 bits
4170 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4171 getF32Constant(DAG, 0x3e814304, dl));
4172 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4173 getF32Constant(DAG, 0x3f3c50c8, dl));
4174 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4175 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4176 getF32Constant(DAG, 0x3f7f5e7e, dl));
4177 } else if (LimitFloatPrecision <= 12) {
4178 // For floating-point precision of 12:
4179 //
4180 // TwoToFractionalPartOfX =
4181 // 0.999892986f +
4182 // (0.696457318f +
4183 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4184 //
4185 // error 0.000107046256, which is 13 to 14 bits
4186 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4187 getF32Constant(DAG, 0x3da235e3, dl));
4188 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4189 getF32Constant(DAG, 0x3e65b8f3, dl));
4190 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4191 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4192 getF32Constant(DAG, 0x3f324b07, dl));
4193 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4194 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4195 getF32Constant(DAG, 0x3f7ff8fd, dl));
4196 } else { // LimitFloatPrecision <= 18
4197 // For floating-point precision of 18:
4198 //
4199 // TwoToFractionalPartOfX =
4200 // 0.999999982f +
4201 // (0.693148872f +
4202 // (0.240227044f +
4203 // (0.554906021e-1f +
4204 // (0.961591928e-2f +
4205 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4206 // error 2.47208000*10^(-7), which is better than 18 bits
4207 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4208 getF32Constant(DAG, 0x3924b03e, dl));
4209 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4210 getF32Constant(DAG, 0x3ab24b87, dl));
4211 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4212 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4213 getF32Constant(DAG, 0x3c1d8c17, dl));
4214 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4215 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4216 getF32Constant(DAG, 0x3d634a1d, dl));
4217 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4218 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4219 getF32Constant(DAG, 0x3e75fe14, dl));
4220 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4221 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4222 getF32Constant(DAG, 0x3f317234, dl));
4223 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4224 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4225 getF32Constant(DAG, 0x3f800000, dl));
4226 }
4227
4228 // Add the exponent into the result in integer domain.
4229 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4230 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4231 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4232}
4233
4234/// expandExp - Lower an exp intrinsic. Handles the special sequences for
4235/// limited-precision mode.
4236static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4237 const TargetLowering &TLI) {
4238 if (Op.getValueType() == MVT::f32 &&
4239 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4240
4241 // Put the exponent in the right bit position for later addition to the
4242 // final result:
4243 //
4244 // #define LOG2OFe 1.4426950f
4245 // t0 = Op * LOG2OFe
4246
4247 // TODO: What fast-math-flags should be set here?
4248 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4249 getF32Constant(DAG, 0x3fb8aa3b, dl));
4250 return getLimitedPrecisionExp2(t0, dl, DAG);
4251 }
4252
4253 // No special expansion.
4254 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4255}
4256
4257/// expandLog - Lower a log intrinsic. Handles the special sequences for
4258/// limited-precision mode.
4259static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4260 const TargetLowering &TLI) {
4261
4262 // TODO: What fast-math-flags should be set on the floating-point nodes?
4263
4264 if (Op.getValueType() == MVT::f32 &&
4265 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4266 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4267
4268 // Scale the exponent by log(2) [0.69314718f].
4269 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4270 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4271 getF32Constant(DAG, 0x3f317218, dl));
4272
4273 // Get the significand and build it into a floating-point number with
4274 // exponent of 1.
4275 SDValue X = GetSignificand(DAG, Op1, dl);
4276
4277 SDValue LogOfMantissa;
4278 if (LimitFloatPrecision <= 6) {
4279 // For floating-point precision of 6:
4280 //
4281 // LogofMantissa =
4282 // -1.1609546f +
4283 // (1.4034025f - 0.23903021f * x) * x;
4284 //
4285 // error 0.0034276066, which is better than 8 bits
4286 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4287 getF32Constant(DAG, 0xbe74c456, dl));
4288 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4289 getF32Constant(DAG, 0x3fb3a2b1, dl));
4290 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4291 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4292 getF32Constant(DAG, 0x3f949a29, dl));
4293 } else if (LimitFloatPrecision <= 12) {
4294 // For floating-point precision of 12:
4295 //
4296 // LogOfMantissa =
4297 // -1.7417939f +
4298 // (2.8212026f +
4299 // (-1.4699568f +
4300 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4301 //
4302 // error 0.000061011436, which is 14 bits
4303 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4304 getF32Constant(DAG, 0xbd67b6d6, dl));
4305 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4306 getF32Constant(DAG, 0x3ee4f4b8, dl));
4307 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4308 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4309 getF32Constant(DAG, 0x3fbc278b, dl));
4310 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4311 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4312 getF32Constant(DAG, 0x40348e95, dl));
4313 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4314 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4315 getF32Constant(DAG, 0x3fdef31a, dl));
4316 } else { // LimitFloatPrecision <= 18
4317 // For floating-point precision of 18:
4318 //
4319 // LogOfMantissa =
4320 // -2.1072184f +
4321 // (4.2372794f +
4322 // (-3.7029485f +
4323 // (2.2781945f +
4324 // (-0.87823314f +
4325 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4326 //
4327 // error 0.0000023660568, which is better than 18 bits
4328 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4329 getF32Constant(DAG, 0xbc91e5ac, dl));
4330 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4331 getF32Constant(DAG, 0x3e4350aa, dl));
4332 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4333 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4334 getF32Constant(DAG, 0x3f60d3e3, dl));
4335 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4336 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4337 getF32Constant(DAG, 0x4011cdf0, dl));
4338 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4339 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4340 getF32Constant(DAG, 0x406cfd1c, dl));
4341 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4342 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4343 getF32Constant(DAG, 0x408797cb, dl));
4344 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4345 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4346 getF32Constant(DAG, 0x4006dcab, dl));
4347 }
4348
4349 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4350 }
4351
4352 // No special expansion.
4353 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4354}
4355
4356/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4357/// limited-precision mode.
4358static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4359 const TargetLowering &TLI) {
4360
4361 // TODO: What fast-math-flags should be set on the floating-point nodes?
4362
4363 if (Op.getValueType() == MVT::f32 &&
4364 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4365 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4366
4367 // Get the exponent.
4368 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4369
4370 // Get the significand and build it into a floating-point number with
4371 // exponent of 1.
4372 SDValue X = GetSignificand(DAG, Op1, dl);
4373
4374 // Different possible minimax approximations of significand in
4375 // floating-point for various degrees of accuracy over [1,2].
4376 SDValue Log2ofMantissa;
4377 if (LimitFloatPrecision <= 6) {
4378 // For floating-point precision of 6:
4379 //
4380 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4381 //
4382 // error 0.0049451742, which is more than 7 bits
4383 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4384 getF32Constant(DAG, 0xbeb08fe0, dl));
4385 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4386 getF32Constant(DAG, 0x40019463, dl));
4387 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4388 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4389 getF32Constant(DAG, 0x3fd6633d, dl));
4390 } else if (LimitFloatPrecision <= 12) {
4391 // For floating-point precision of 12:
4392 //
4393 // Log2ofMantissa =
4394 // -2.51285454f +
4395 // (4.07009056f +
4396 // (-2.12067489f +
4397 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4398 //
4399 // error 0.0000876136000, which is better than 13 bits
4400 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4401 getF32Constant(DAG, 0xbda7262e, dl));
4402 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4403 getF32Constant(DAG, 0x3f25280b, dl));
4404 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4405 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4406 getF32Constant(DAG, 0x4007b923, dl));
4407 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4408 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4409 getF32Constant(DAG, 0x40823e2f, dl));
4410 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4411 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4412 getF32Constant(DAG, 0x4020d29c, dl));
4413 } else { // LimitFloatPrecision <= 18
4414 // For floating-point precision of 18:
4415 //
4416 // Log2ofMantissa =
4417 // -3.0400495f +
4418 // (6.1129976f +
4419 // (-5.3420409f +
4420 // (3.2865683f +
4421 // (-1.2669343f +
4422 // (0.27515199f -
4423 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4424 //
4425 // error 0.0000018516, which is better than 18 bits
4426 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4427 getF32Constant(DAG, 0xbcd2769e, dl));
4428 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4429 getF32Constant(DAG, 0x3e8ce0b9, dl));
4430 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4431 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4432 getF32Constant(DAG, 0x3fa22ae7, dl));
4433 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4434 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4435 getF32Constant(DAG, 0x40525723, dl));
4436 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4437 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4438 getF32Constant(DAG, 0x40aaf200, dl));
4439 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4440 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4441 getF32Constant(DAG, 0x40c39dad, dl));
4442 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4443 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4444 getF32Constant(DAG, 0x4042902c, dl));
4445 }
4446
4447 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4448 }
4449
4450 // No special expansion.
4451 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4452}
4453
4454/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4455/// limited-precision mode.
4456static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4457 const TargetLowering &TLI) {
4458
4459 // TODO: What fast-math-flags should be set on the floating-point nodes?
4460
4461 if (Op.getValueType() == MVT::f32 &&
4462 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4463 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4464
4465 // Scale the exponent by log10(2) [0.30102999f].
4466 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4467 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4468 getF32Constant(DAG, 0x3e9a209a, dl));
4469
4470 // Get the significand and build it into a floating-point number with
4471 // exponent of 1.
4472 SDValue X = GetSignificand(DAG, Op1, dl);
4473
4474 SDValue Log10ofMantissa;
4475 if (LimitFloatPrecision <= 6) {
4476 // For floating-point precision of 6:
4477 //
4478 // Log10ofMantissa =
4479 // -0.50419619f +
4480 // (0.60948995f - 0.10380950f * x) * x;
4481 //
4482 // error 0.0014886165, which is 6 bits
4483 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4484 getF32Constant(DAG, 0xbdd49a13, dl));
4485 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4486 getF32Constant(DAG, 0x3f1c0789, dl));
4487 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4488 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4489 getF32Constant(DAG, 0x3f011300, dl));
4490 } else if (LimitFloatPrecision <= 12) {
4491 // For floating-point precision of 12:
4492 //
4493 // Log10ofMantissa =
4494 // -0.64831180f +
4495 // (0.91751397f +
4496 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4497 //
4498 // error 0.00019228036, which is better than 12 bits
4499 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4500 getF32Constant(DAG, 0x3d431f31, dl));
4501 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4502 getF32Constant(DAG, 0x3ea21fb2, dl));
4503 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4504 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4505 getF32Constant(DAG, 0x3f6ae232, dl));
4506 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4507 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4508 getF32Constant(DAG, 0x3f25f7c3, dl));
4509 } else { // LimitFloatPrecision <= 18
4510 // For floating-point precision of 18:
4511 //
4512 // Log10ofMantissa =
4513 // -0.84299375f +
4514 // (1.5327582f +
4515 // (-1.0688956f +
4516 // (0.49102474f +
4517 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4518 //
4519 // error 0.0000037995730, which is better than 18 bits
4520 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4521 getF32Constant(DAG, 0x3c5d51ce, dl));
4522 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4523 getF32Constant(DAG, 0x3e00685a, dl));
4524 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4525 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4526 getF32Constant(DAG, 0x3efb6798, dl));
4527 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4528 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4529 getF32Constant(DAG, 0x3f88d192, dl));
4530 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4531 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4532 getF32Constant(DAG, 0x3fc4316c, dl));
4533 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4534 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4535 getF32Constant(DAG, 0x3f57ce70, dl));
4536 }
4537
4538 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4539 }
4540
4541 // No special expansion.
4542 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4543}
4544
4545/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4546/// limited-precision mode.
4547static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4548 const TargetLowering &TLI) {
4549 if (Op.getValueType() == MVT::f32 &&
4550 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4551 return getLimitedPrecisionExp2(Op, dl, DAG);
4552
4553 // No special expansion.
4554 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4555}
4556
4557/// visitPow - Lower a pow intrinsic. Handles the special sequences for
4558/// limited-precision mode with x == 10.0f.
4559static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4560 SelectionDAG &DAG, const TargetLowering &TLI) {
4561 bool IsExp10 = false;
4562 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4563 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4564 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4565 APFloat Ten(10.0f);
4566 IsExp10 = LHSC->isExactlyValue(Ten);
4567 }
4568 }
4569
4570 // TODO: What fast-math-flags should be set on the FMUL node?
4571 if (IsExp10) {
4572 // Put the exponent in the right bit position for later addition to the
4573 // final result:
4574 //
4575 // #define LOG2OF10 3.3219281f
4576 // t0 = Op * LOG2OF10;
4577 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4578 getF32Constant(DAG, 0x40549a78, dl));
4579 return getLimitedPrecisionExp2(t0, dl, DAG);
4580 }
4581
4582 // No special expansion.
4583 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4584}
4585
4586
4587/// ExpandPowI - Expand a llvm.powi intrinsic.
4588static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4589 SelectionDAG &DAG) {
4590 // If RHS is a constant, we can expand this out to a multiplication tree,
4591 // otherwise we end up lowering to a call to __powidf2 (for example). When
4592 // optimizing for size, we only want to do this if the expansion would produce
4593 // a small number of multiplies, otherwise we do the full expansion.
4594 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4595 // Get the exponent as a positive value.
4596 unsigned Val = RHSC->getSExtValue();
4597 if ((int)Val < 0) Val = -Val;
4598
4599 // powi(x, 0) -> 1.0
4600 if (Val == 0)
4601 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4602
4603 const Function *F = DAG.getMachineFunction().getFunction();
4604 if (!F->optForSize() ||
4605 // If optimizing for size, don't insert too many multiplies.
4606 // This inserts up to 5 multiplies.
4607 countPopulation(Val) + Log2_32(Val) < 7) {
4608 // We use the simple binary decomposition method to generate the multiply
4609 // sequence. There are more optimal ways to do this (for example,
4610 // powi(x,15) generates one more multiply than it should), but this has
4611 // the benefit of being both really simple and much better than a libcall.
4612 SDValue Res; // Logically starts equal to 1.0
4613 SDValue CurSquare = LHS;
4614 // TODO: Intrinsics should have fast-math-flags that propagate to these
4615 // nodes.
4616 while (Val) {
4617 if (Val & 1) {
4618 if (Res.getNode())
4619 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4620 else
4621 Res = CurSquare; // 1.0*CurSquare.
4622 }
4623
4624 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4625 CurSquare, CurSquare);
4626 Val >>= 1;
4627 }
4628
4629 // If the original was negative, invert the result, producing 1/(x*x*x).
4630 if (RHSC->getSExtValue() < 0)
4631 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4632 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4633 return Res;
4634 }
4635 }
4636
4637 // Otherwise, expand to a libcall.
4638 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4639}
4640
4641// getUnderlyingArgReg - Find underlying register used for a truncated or
4642// bitcasted argument.
4643static unsigned getUnderlyingArgReg(const SDValue &N) {
4644 switch (N.getOpcode()) {
4645 case ISD::CopyFromReg:
4646 return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4647 case ISD::BITCAST:
4648 case ISD::AssertZext:
4649 case ISD::AssertSext:
4650 case ISD::TRUNCATE:
4651 return getUnderlyingArgReg(N.getOperand(0));
4652 default:
4653 return 0;
4654 }
4655}
4656
4657/// EmitFuncArgumentDbgValue - If the DbgValueInst is a dbg_value of a function
4658/// argument, create the corresponding DBG_VALUE machine instruction for it now.
4659/// At the end of instruction selection, they will be inserted to the entry BB.
4660bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4661 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4662 DILocation *DL, int64_t Offset, bool IsDbgDeclare, const SDValue &N) {
4663 const Argument *Arg = dyn_cast<Argument>(V);
4664 if (!Arg)
4665 return false;
4666
4667 MachineFunction &MF = DAG.getMachineFunction();
4668 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4669
4670 // Ignore inlined function arguments here.
4671 //
4672 // FIXME: Should we be checking DL->inlinedAt() to determine this?
4673 if (!Variable->getScope()->getSubprogram()->describes(MF.getFunction()))
4674 return false;
4675
4676 bool IsIndirect = false;
4677 Optional<MachineOperand> Op;
4678 // Some arguments' frame index is recorded during argument lowering.
4679 int FI = FuncInfo.getArgumentFrameIndex(Arg);
4680 if (FI != INT_MAX2147483647)
4681 Op = MachineOperand::CreateFI(FI);
4682
4683 if (!Op && N.getNode()) {
4684 unsigned Reg = getUnderlyingArgReg(N);
4685 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4686 MachineRegisterInfo &RegInfo = MF.getRegInfo();
4687 unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4688 if (PR)
4689 Reg = PR;
4690 }
4691 if (Reg) {
4692 Op = MachineOperand::CreateReg(Reg, false);
4693 IsIndirect = IsDbgDeclare;
4694 }
4695 }
4696
4697 if (!Op) {
4698 // Check if ValueMap has reg number.
4699 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4700 if (VMI != FuncInfo.ValueMap.end()) {
4701 Op = MachineOperand::CreateReg(VMI->second, false);
4702 IsIndirect = IsDbgDeclare;
4703 }
4704 }
4705
4706 if (!Op && N.getNode())
4707 // Check if frame index is available.
4708 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4709 if (FrameIndexSDNode *FINode =
4710 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4711 Op = MachineOperand::CreateFI(FINode->getIndex());
4712
4713 if (!Op)
4714 return false;
4715
4716 assert(Variable->isValidLocationForIntrinsic(DL) &&((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4717, __PRETTY_FUNCTION__))
4717 "Expected inlined-at fields to agree")((Variable->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"
) ? static_cast<void> (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4717, __PRETTY_FUNCTION__))
;
4718 if (Op->isReg())
4719 FuncInfo.ArgDbgValues.push_back(
4720 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4721 Op->getReg(), Offset, Variable, Expr));
4722 else
4723 FuncInfo.ArgDbgValues.push_back(
4724 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE))
4725 .add(*Op)
4726 .addImm(Offset)
4727 .addMetadata(Variable)
4728 .addMetadata(Expr));
4729
4730 return true;
4731}
4732
4733/// Return the appropriate SDDbgValue based on N.
4734SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4735 DILocalVariable *Variable,
4736 DIExpression *Expr, int64_t Offset,
4737 const DebugLoc &dl,
4738 unsigned DbgSDNodeOrder) {
4739 SDDbgValue *SDV;
4740 auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode());
4741 if (FISDN && Expr->startsWithDeref()) {
4742 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4743 // stack slot locations as such instead of as indirectly addressed
4744 // locations.
4745 ArrayRef<uint64_t> TrailingElements(Expr->elements_begin() + 1,
4746 Expr->elements_end());
4747 DIExpression *DerefedDIExpr =
4748 DIExpression::get(*DAG.getContext(), TrailingElements);
4749 int FI = FISDN->getIndex();
4750 SDV = DAG.getFrameIndexDbgValue(Variable, DerefedDIExpr, FI, 0, dl,
4751 DbgSDNodeOrder);
4752 } else {
4753 SDV = DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), false,
4754 Offset, dl, DbgSDNodeOrder);
4755 }
4756 return SDV;
4757}
4758
4759// VisualStudio defines setjmp as _setjmp
4760#if defined(_MSC_VER) && defined(setjmp) && \
4761 !defined(setjmp_undefined_for_msvc)
4762# pragma push_macro("setjmp")
4763# undef setjmp
4764# define setjmp_undefined_for_msvc
4765#endif
4766
4767/// Lower the call to the specified intrinsic function. If we want to emit this
4768/// as a call to a named external function, return the name. Otherwise, lower it
4769/// and return null.
4770const char *
4771SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
4772 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4773 SDLoc sdl = getCurSDLoc();
4774 DebugLoc dl = getCurDebugLoc();
4775 SDValue Res;
4776
4777 switch (Intrinsic) {
4778 default:
4779 // By default, turn this into a target intrinsic node.
4780 visitTargetIntrinsic(I, Intrinsic);
4781 return nullptr;
4782 case Intrinsic::vastart: visitVAStart(I); return nullptr;
4783 case Intrinsic::vaend: visitVAEnd(I); return nullptr;
4784 case Intrinsic::vacopy: visitVACopy(I); return nullptr;
4785 case Intrinsic::returnaddress:
4786 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
4787 TLI.getPointerTy(DAG.getDataLayout()),
4788 getValue(I.getArgOperand(0))));
4789 return nullptr;
4790 case Intrinsic::addressofreturnaddress:
4791 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
4792 TLI.getPointerTy(DAG.getDataLayout())));
4793 return nullptr;
4794 case Intrinsic::frameaddress:
4795 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
4796 TLI.getPointerTy(DAG.getDataLayout()),
4797 getValue(I.getArgOperand(0))));
4798 return nullptr;
4799 case Intrinsic::read_register: {
4800 Value *Reg = I.getArgOperand(0);
4801 SDValue Chain = getRoot();
4802 SDValue RegName =
4803 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4804 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4805 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
4806 DAG.getVTList(VT, MVT::Other), Chain, RegName);
4807 setValue(&I, Res);
4808 DAG.setRoot(Res.getValue(1));
4809 return nullptr;
4810 }
4811 case Intrinsic::write_register: {
4812 Value *Reg = I.getArgOperand(0);
4813 Value *RegValue = I.getArgOperand(1);
4814 SDValue Chain = getRoot();
4815 SDValue RegName =
4816 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
4817 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
4818 RegName, getValue(RegValue)));
4819 return nullptr;
4820 }
4821 case Intrinsic::setjmp:
4822 return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
4823 case Intrinsic::longjmp:
4824 return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
4825 case Intrinsic::memcpy: {
4826 SDValue Op1 = getValue(I.getArgOperand(0));
4827 SDValue Op2 = getValue(I.getArgOperand(1));
4828 SDValue Op3 = getValue(I.getArgOperand(2));
4829 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4830 if (!Align)
4831 Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
4832 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4833 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4834 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4835 false, isTC,
4836 MachinePointerInfo(I.getArgOperand(0)),
4837 MachinePointerInfo(I.getArgOperand(1)));
4838 updateDAGForMaybeTailCall(MC);
4839 return nullptr;
4840 }
4841 case Intrinsic::memset: {
4842 SDValue Op1 = getValue(I.getArgOperand(0));
4843 SDValue Op2 = getValue(I.getArgOperand(1));
4844 SDValue Op3 = getValue(I.getArgOperand(2));
4845 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4846 if (!Align)
4847 Align = 1; // @llvm.memset defines 0 and 1 to both mean no alignment.
4848 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4849 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4850 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4851 isTC, MachinePointerInfo(I.getArgOperand(0)));
4852 updateDAGForMaybeTailCall(MS);
4853 return nullptr;
4854 }
4855 case Intrinsic::memmove: {
4856 SDValue Op1 = getValue(I.getArgOperand(0));
4857 SDValue Op2 = getValue(I.getArgOperand(1));
4858 SDValue Op3 = getValue(I.getArgOperand(2));
4859 unsigned Align = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4860 if (!Align)
4861 Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
4862 bool isVol = cast<ConstantInt>(I.getArgOperand(4))->getZExtValue();
4863 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
4864 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
4865 isTC, MachinePointerInfo(I.getArgOperand(0)),
4866 MachinePointerInfo(I.getArgOperand(1)));
4867 updateDAGForMaybeTailCall(MM);
4868 return nullptr;
4869 }
4870 case Intrinsic::memcpy_element_atomic: {
4871 SDValue Dst = getValue(I.getArgOperand(0));
4872 SDValue Src = getValue(I.getArgOperand(1));
4873 SDValue NumElements = getValue(I.getArgOperand(2));
4874 SDValue ElementSize = getValue(I.getArgOperand(3));
4875
4876 // Emit a library call.
4877 TargetLowering::ArgListTy Args;
4878 TargetLowering::ArgListEntry Entry;
4879 Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
4880 Entry.Node = Dst;
4881 Args.push_back(Entry);
4882
4883 Entry.Node = Src;
4884 Args.push_back(Entry);
4885
4886 Entry.Ty = I.getArgOperand(2)->getType();
4887 Entry.Node = NumElements;
4888 Args.push_back(Entry);
4889
4890 Entry.Ty = Type::getInt32Ty(*DAG.getContext());
4891 Entry.Node = ElementSize;
4892 Args.push_back(Entry);
4893
4894 uint64_t ElementSizeConstant =
4895 cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
4896 RTLIB::Libcall LibraryCall =
4897 RTLIB::getMEMCPY_ELEMENT_ATOMIC(ElementSizeConstant);
4898 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
4899 report_fatal_error("Unsupported element size");
4900
4901 TargetLowering::CallLoweringInfo CLI(DAG);
4902 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
4903 TLI.getLibcallCallingConv(LibraryCall),
4904 Type::getVoidTy(*DAG.getContext()),
4905 DAG.getExternalSymbol(TLI.getLibcallName(LibraryCall),
4906 TLI.getPointerTy(DAG.getDataLayout())),
4907 std::move(Args));
4908
4909 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4910 DAG.setRoot(CallResult.second);
4911 return nullptr;
4912 }
4913 case Intrinsic::dbg_declare: {
4914 const DbgDeclareInst &DI = cast<DbgDeclareInst>(I);
4915 DILocalVariable *Variable = DI.getVariable();
4916 DIExpression *Expression = DI.getExpression();
4917 const Value *Address = DI.getAddress();
4918 assert(Variable && "Missing variable")((Variable && "Missing variable") ? static_cast<void
> (0) : __assert_fail ("Variable && \"Missing variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4918, __PRETTY_FUNCTION__))
;
4919 if (!Address) {
4920 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
4921 return nullptr;
4922 }
4923
4924 // Check if address has undef value.
4925 if (isa<UndefValue>(Address) ||
4926 (Address->use_empty() && !isa<Argument>(Address))) {
4927 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
4928 return nullptr;
4929 }
4930
4931 // Byval arguments with frame indices were already handled after argument
4932 // lowering and before isel.
4933 const auto *Arg =
4934 dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
4935 if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX2147483647)
4936 return nullptr;
4937
4938 SDValue &N = NodeMap[Address];
4939 if (!N.getNode() && isa<Argument>(Address))
4940 // Check unused arguments map.
4941 N = UnusedArgNodeMap[Address];
4942 SDDbgValue *SDV;
4943 if (N.getNode()) {
4944 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
4945 Address = BCI->getOperand(0);
4946 // Parameters are handled specially.
4947 bool isParameter = Variable->isParameter() || isa<Argument>(Address);
4948 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
4949 if (isParameter && FINode) {
4950 // Byval parameter. We have a frame index at this point.
4951 SDV = DAG.getFrameIndexDbgValue(Variable, Expression,
4952 FINode->getIndex(), 0, dl, SDNodeOrder);
4953 } else if (isa<Argument>(Address)) {
4954 // Address is an argument, so try to emit its dbg value using
4955 // virtual register info from the FuncInfo.ValueMap.
4956 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, true, N);
4957 return nullptr;
4958 } else {
4959 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
4960 true, 0, dl, SDNodeOrder);
4961 }
4962 DAG.AddDbgValue(SDV, N.getNode(), isParameter);
4963 } else {
4964 // If Address is an argument then try to emit its dbg value using
4965 // virtual register info from the FuncInfo.ValueMap.
4966 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 0, true,
4967 N)) {
4968 DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false)
;
4969 }
4970 }
4971 return nullptr;
4972 }
4973 case Intrinsic::dbg_value: {
4974 const DbgValueInst &DI = cast<DbgValueInst>(I);
4975 assert(DI.getVariable() && "Missing variable")((DI.getVariable() && "Missing variable") ? static_cast
<void> (0) : __assert_fail ("DI.getVariable() && \"Missing variable\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4975, __PRETTY_FUNCTION__))
;
4976
4977 DILocalVariable *Variable = DI.getVariable();
4978 DIExpression *Expression = DI.getExpression();
4979 uint64_t Offset = DI.getOffset();
4980 const Value *V = DI.getValue();
4981 if (!V)
4982 return nullptr;
4983
4984 SDDbgValue *SDV;
4985 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
4986 SDV = DAG.getConstantDbgValue(Variable, Expression, V, Offset, dl,
4987 SDNodeOrder);
4988 DAG.AddDbgValue(SDV, nullptr, false);
4989 return nullptr;
4990 }
4991
4992 // Do not use getValue() in here; we don't want to generate code at
4993 // this point if it hasn't been done yet.
4994 SDValue N = NodeMap[V];
4995 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
4996 N = UnusedArgNodeMap[V];
4997 if (N.getNode()) {
4998 if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, Offset, false,
4999 N))
5000 return nullptr;
5001 SDV = getDbgValue(N, Variable, Expression, Offset, dl, SDNodeOrder);
5002 DAG.AddDbgValue(SDV, N.getNode(), false);
5003 return nullptr;
5004 }
5005
5006 if (!V->use_empty() ) {
5007 // Do not call getValue(V) yet, as we don't want to generate code.
5008 // Remember it for later.
5009 DanglingDebugInfo DDI(&DI, dl, SDNodeOrder);
5010 DanglingDebugInfoMap[V] = DDI;
5011 return nullptr;
5012 }
5013
5014 DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug location info for:\n "
<< DI << "\n"; } } while (false)
;
5015 DEBUG(dbgs() << " Last seen at:\n " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
V << "\n"; } } while (false)
;
5016 return nullptr;
5017 }
5018
5019 case Intrinsic::eh_typeid_for: {
5020 // Find the type id for the given typeinfo.
5021 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5022 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5023 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5024 setValue(&I, Res);
5025 return nullptr;
5026 }
5027
5028 case Intrinsic::eh_return_i32:
5029 case Intrinsic::eh_return_i64:
5030 DAG.getMachineFunction().setCallsEHReturn(true);
5031 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5032 MVT::Other,
5033 getControlRoot(),
5034 getValue(I.getArgOperand(0)),
5035 getValue(I.getArgOperand(1))));
5036 return nullptr;
5037 case Intrinsic::eh_unwind_init:
5038 DAG.getMachineFunction().setCallsUnwindInit(true);
5039 return nullptr;
5040 case Intrinsic::eh_dwarf_cfa: {
5041 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5042 TLI.getPointerTy(DAG.getDataLayout()),
5043 getValue(I.getArgOperand(0))));
5044 return nullptr;
5045 }
5046 case Intrinsic::eh_sjlj_callsite: {
5047 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5048 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5049 assert(CI && "Non-constant call site value in eh.sjlj.callsite!")((CI && "Non-constant call site value in eh.sjlj.callsite!"
) ? static_cast<void> (0) : __assert_fail ("CI && \"Non-constant call site value in eh.sjlj.callsite!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5049, __PRETTY_FUNCTION__))
;
5050 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")((MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"
) ? static_cast<void> (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5050, __PRETTY_FUNCTION__))
;
5051
5052 MMI.setCurrentCallSite(CI->getZExtValue());
5053 return nullptr;
5054 }
5055 case Intrinsic::eh_sjlj_functioncontext: {
5056 // Get and store the index of the function context.
5057 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5058 AllocaInst *FnCtx =
5059 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5060 int FI = FuncInfo.StaticAllocaMap[FnCtx];
5061 MFI.setFunctionContextIndex(FI);
5062 return nullptr;
5063 }
5064 case Intrinsic::eh_sjlj_setjmp: {
5065 SDValue Ops[2];
5066 Ops[0] = getRoot();
5067 Ops[1] = getValue(I.getArgOperand(0));
5068 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5069 DAG.getVTList(MVT::i32, MVT::Other), Ops);
5070 setValue(&I, Op.getValue(0));
5071 DAG.setRoot(Op.getValue(1));
5072 return nullptr;
5073 }
5074 case Intrinsic::eh_sjlj_longjmp: {
5075 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5076 getRoot(), getValue(I.getArgOperand(0))));
5077 return nullptr;
5078 }
5079 case Intrinsic::eh_sjlj_setup_dispatch: {
5080 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5081 getRoot()));
5082 return nullptr;
5083 }
5084
5085 case Intrinsic::masked_gather:
5086 visitMaskedGather(I);
5087 return nullptr;
5088 case Intrinsic::masked_load:
5089 visitMaskedLoad(I);
5090 return nullptr;
5091 case Intrinsic::masked_scatter:
5092 visitMaskedScatter(I);
5093 return nullptr;
5094 case Intrinsic::masked_store:
5095 visitMaskedStore(I);
5096 return nullptr;
5097 case Intrinsic::masked_expandload:
5098 visitMaskedLoad(I, true /* IsExpanding */);
5099 return nullptr;
5100 case Intrinsic::masked_compressstore:
5101 visitMaskedStore(I, true /* IsCompressing */);
5102 return nullptr;
5103 case Intrinsic::x86_mmx_pslli_w:
5104 case Intrinsic::x86_mmx_pslli_d:
5105 case Intrinsic::x86_mmx_pslli_q:
5106 case Intrinsic::x86_mmx_psrli_w:
5107 case Intrinsic::x86_mmx_psrli_d:
5108 case Intrinsic::x86_mmx_psrli_q:
5109 case Intrinsic::x86_mmx_psrai_w:
5110 case Intrinsic::x86_mmx_psrai_d: {
5111 SDValue ShAmt = getValue(I.getArgOperand(1));
5112 if (isa<ConstantSDNode>(ShAmt)) {
5113 visitTargetIntrinsic(I, Intrinsic);
5114 return nullptr;
5115 }
5116 unsigned NewIntrinsic = 0;
5117 EVT ShAmtVT = MVT::v2i32;
5118 switch (Intrinsic) {
5119 case Intrinsic::x86_mmx_pslli_w:
5120 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5121 break;
5122 case Intrinsic::x86_mmx_pslli_d:
5123 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5124 break;
5125 case Intrinsic::x86_mmx_pslli_q:
5126 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5127 break;
5128 case Intrinsic::x86_mmx_psrli_w:
5129 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5130 break;
5131 case Intrinsic::x86_mmx_psrli_d:
5132 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5133 break;
5134 case Intrinsic::x86_mmx_psrli_q:
5135 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5136 break;
5137 case Intrinsic::x86_mmx_psrai_w:
5138 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5139 break;
5140 case Intrinsic::x86_mmx_psrai_d:
5141 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5142 break;
5143 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5143)
; // Can't reach here.
5144 }
5145
5146 // The vector shift intrinsics with scalars uses 32b shift amounts but
5147 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5148 // to be zero.
5149 // We must do this early because v2i32 is not a legal type.
5150 SDValue ShOps[2];
5151 ShOps[0] = ShAmt;
5152 ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5153 ShAmt = DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5154 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5155 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5156 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5157 DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5158 getValue(I.getArgOperand(0)), ShAmt);
5159 setValue(&I, Res);
5160 return nullptr;
5161 }
5162 case Intrinsic::powi:
5163 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5164 getValue(I.getArgOperand(1)), DAG));
5165 return nullptr;
5166 case Intrinsic::log:
5167 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5168 return nullptr;
5169 case Intrinsic::log2:
5170 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5171 return nullptr;
5172 case Intrinsic::log10:
5173 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5174 return nullptr;
5175 case Intrinsic::exp:
5176 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5177 return nullptr;
5178 case Intrinsic::exp2:
5179 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5180 return nullptr;
5181 case Intrinsic::pow:
5182 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5183 getValue(I.getArgOperand(1)), DAG, TLI));
5184 return nullptr;
5185 case Intrinsic::sqrt:
5186 case Intrinsic::fabs:
5187 case Intrinsic::sin:
5188 case Intrinsic::cos:
5189 case Intrinsic::floor:
5190 case Intrinsic::ceil:
5191 case Intrinsic::trunc:
5192 case Intrinsic::rint:
5193 case Intrinsic::nearbyint:
5194 case Intrinsic::round:
5195 case Intrinsic::canonicalize: {
5196 unsigned Opcode;
5197 switch (Intrinsic) {
5198 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5198)
; // Can't reach here.
5199 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5200 case Intrinsic::fabs: Opcode = ISD::FABS; break;
5201 case Intrinsic::sin: Opcode = ISD::FSIN; break;
5202 case Intrinsic::cos: Opcode = ISD::FCOS; break;
5203 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5204 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5205 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5206 case Intrinsic::rint: Opcode = ISD::FRINT; break;
5207 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5208 case Intrinsic::round: Opcode = ISD::FROUND; break;
5209 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5210 }
5211
5212 setValue(&I, DAG.getNode(Opcode, sdl,
5213 getValue(I.getArgOperand(0)).getValueType(),
5214 getValue(I.getArgOperand(0))));
5215 return nullptr;
5216 }
5217 case Intrinsic::minnum: {
5218 auto VT = getValue(I.getArgOperand(0)).getValueType();
5219 unsigned Opc =
5220 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT)
5221 ? ISD::FMINNAN
5222 : ISD::FMINNUM;
5223 setValue(&I, DAG.getNode(Opc, sdl, VT,
5224 getValue(I.getArgOperand(0)),
5225 getValue(I.getArgOperand(1))));
5226 return nullptr;
5227 }
5228 case Intrinsic::maxnum: {
5229 auto VT = getValue(I.getArgOperand(0)).getValueType();
5230 unsigned Opc =
5231 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT)
5232 ? ISD::FMAXNAN
5233 : ISD::FMAXNUM;
5234 setValue(&I, DAG.getNode(Opc, sdl, VT,
5235 getValue(I.getArgOperand(0)),
5236 getValue(I.getArgOperand(1))));
5237 return nullptr;
5238 }
5239 case Intrinsic::copysign:
5240 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5241 getValue(I.getArgOperand(0)).getValueType(),
5242 getValue(I.getArgOperand(0)),
5243 getValue(I.getArgOperand(1))));
5244 return nullptr;
5245 case Intrinsic::fma:
5246 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5247 getValue(I.getArgOperand(0)).getValueType(),
5248 getValue(I.getArgOperand(0)),
5249 getValue(I.getArgOperand(1)),
5250 getValue(I.getArgOperand(2))));
5251 return nullptr;
5252 case Intrinsic::experimental_constrained_fadd:
5253 case Intrinsic::experimental_constrained_fsub:
5254 case Intrinsic::experimental_constrained_fmul:
5255 case Intrinsic::experimental_constrained_fdiv:
5256 case Intrinsic::experimental_constrained_frem:
5257 visitConstrainedFPIntrinsic(I, Intrinsic);
5258 return nullptr;
5259 case Intrinsic::fmuladd: {
5260 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5261 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5262 TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5263 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5264 getValue(I.getArgOperand(0)).getValueType(),
5265 getValue(I.getArgOperand(0)),
5266 getValue(I.getArgOperand(1)),
5267 getValue(I.getArgOperand(2))));
5268 } else {
5269 // TODO: Intrinsic calls should have fast-math-flags.
5270 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5271 getValue(I.getArgOperand(0)).getValueType(),
5272 getValue(I.getArgOperand(0)),
5273 getValue(I.getArgOperand(1)));
5274 SDValue Add = DAG.getNode(ISD::FADD, sdl,
5275 getValue(I.getArgOperand(0)).getValueType(),
5276 Mul,
5277 getValue(I.getArgOperand(2)));
5278 setValue(&I, Add);
5279 }
5280 return nullptr;
5281 }
5282 case Intrinsic::convert_to_fp16:
5283 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5284 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5285 getValue(I.getArgOperand(0)),
5286 DAG.getTargetConstant(0, sdl,
5287 MVT::i32))));
5288 return nullptr;
5289 case Intrinsic::convert_from_fp16:
5290 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5291 TLI.getValueType(DAG.getDataLayout(), I.getType()),
5292 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5293 getValue(I.getArgOperand(0)))));
5294 return nullptr;
5295 case Intrinsic::pcmarker: {
5296 SDValue Tmp = getValue(I.getArgOperand(0));
5297 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5298 return nullptr;
5299 }
5300 case Intrinsic::readcyclecounter: {
5301 SDValue Op = getRoot();
5302 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5303 DAG.getVTList(MVT::i64, MVT::Other), Op);
5304 setValue(&I, Res);
5305 DAG.setRoot(Res.getValue(1));
5306 return nullptr;
5307 }
5308 case Intrinsic::bitreverse:
5309 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5310 getValue(I.getArgOperand(0)).getValueType(),
5311 getValue(I.getArgOperand(0))));
5312 return nullptr;
5313 case Intrinsic::bswap:
5314 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5315 getValue(I.getArgOperand(0)).getValueType(),
5316 getValue(I.getArgOperand(0))));
5317 return nullptr;
5318 case Intrinsic::cttz: {
5319 SDValue Arg = getValue(I.getArgOperand(0));
5320 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5321 EVT Ty = Arg.getValueType();
5322 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5323 sdl, Ty, Arg));
5324 return nullptr;
5325 }
5326 case Intrinsic::ctlz: {
5327 SDValue Arg = getValue(I.getArgOperand(0));
5328 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5329 EVT Ty = Arg.getValueType();
5330 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5331 sdl, Ty, Arg));
5332 return nullptr;
5333 }
5334 case Intrinsic::ctpop: {
5335 SDValue Arg = getValue(I.getArgOperand(0));
5336 EVT Ty = Arg.getValueType();
5337 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5338 return nullptr;
5339 }
5340 case Intrinsic::stacksave: {
5341 SDValue Op = getRoot();
5342 Res = DAG.getNode(
5343 ISD::STACKSAVE, sdl,
5344 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5345 setValue(&I, Res);
5346 DAG.setRoot(Res.getValue(1));
5347 return nullptr;
5348 }
5349 case Intrinsic::stackrestore: {
5350 Res = getValue(I.getArgOperand(0));
5351 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5352 return nullptr;
5353 }
5354 case Intrinsic::get_dynamic_area_offset: {
5355 SDValue Op = getRoot();
5356 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5357 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5358 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5359 // target.
5360 if (PtrTy != ResTy)
5361 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5362 " intrinsic!");
5363 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5364 Op);
5365 DAG.setRoot(Op);
5366 setValue(&I, Res);
5367 return nullptr;
5368 }
5369 case Intrinsic::stackguard: {
5370 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5371 MachineFunction &MF = DAG.getMachineFunction();
5372 const Module &M = *MF.getFunction()->getParent();
5373 SDValue Chain = getRoot();
5374 if (TLI.useLoadStackGuardNode()) {
5375 Res = getLoadStackGuard(DAG, sdl, Chain);
5376 } else {
5377 const Value *Global = TLI.getSDagStackGuard(M);
5378 unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5379 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5380 MachinePointerInfo(Global, 0), Align,
5381 MachineMemOperand::MOVolatile);
5382 }
5383 DAG.setRoot(Chain);
5384 setValue(&I, Res);
5385 return nullptr;
5386 }
5387 case Intrinsic::stackprotector: {
5388 // Emit code into the DAG to store the stack guard onto the stack.
5389 MachineFunction &MF = DAG.getMachineFunction();
5390 MachineFrameInfo &MFI = MF.getFrameInfo();
5391 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5392 SDValue Src, Chain = getRoot();
5393
5394 if (TLI.useLoadStackGuardNode())
5395 Src = getLoadStackGuard(DAG, sdl, Chain);
5396 else
5397 Src = getValue(I.getArgOperand(0)); // The guard's value.
5398
5399 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5400
5401 int FI = FuncInfo.StaticAllocaMap[Slot];
5402 MFI.setStackProtectorIndex(FI);
5403
5404 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5405
5406 // Store the stack protector onto the stack.
5407 Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5408 DAG.getMachineFunction(), FI),
5409 /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5410 setValue(&I, Res);
5411 DAG.setRoot(Res);
5412 return nullptr;
5413 }
5414 case Intrinsic::objectsize: {
5415 // If we don't know by now, we're never going to know.
5416 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5417
5418 assert(CI && "Non-constant type in __builtin_object_size?")((CI && "Non-constant type in __builtin_object_size?"
) ? static_cast<void> (0) : __assert_fail ("CI && \"Non-constant type in __builtin_object_size?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5418, __PRETTY_FUNCTION__))
;
5419
5420 SDValue Arg = getValue(I.getCalledValue());
5421 EVT Ty = Arg.getValueType();
5422
5423 if (CI->isZero())
5424 Res = DAG.getConstant(-1ULL, sdl, Ty);
5425 else
5426 Res = DAG.getConstant(0, sdl, Ty);
5427
5428 setValue(&I, Res);
5429 return nullptr;
5430 }
5431 case Intrinsic::annotation:
5432 case Intrinsic::ptr_annotation:
5433 case Intrinsic::invariant_group_barrier:
5434 // Drop the intrinsic, but forward the value
5435 setValue(&I, getValue(I.getOperand(0)));
5436 return nullptr;
5437 case Intrinsic::assume:
5438 case Intrinsic::var_annotation:
5439 // Discard annotate attributes and assumptions
5440 return nullptr;
5441
5442 case Intrinsic::init_trampoline: {
5443 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5444
5445 SDValue Ops[6];
5446 Ops[0] = getRoot();
5447 Ops[1] = getValue(I.getArgOperand(0));
5448 Ops[2] = getValue(I.getArgOperand(1));
5449 Ops[3] = getValue(I.getArgOperand(2));
5450 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5451 Ops[5] = DAG.getSrcValue(F);
5452
5453 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5454
5455 DAG.setRoot(Res);
5456 return nullptr;
5457 }
5458 case Intrinsic::adjust_trampoline: {
5459 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5460 TLI.getPointerTy(DAG.getDataLayout()),
5461 getValue(I.getArgOperand(0))));
5462 return nullptr;
5463 }
5464 case Intrinsic::gcroot: {
5465 MachineFunction &MF = DAG.getMachineFunction();
5466 const Function *F = MF.getFunction();
5467 (void)F;
5468 assert(F->hasGC() &&((F->hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? static_cast<void> (0) : __assert_fail ("F->hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5469, __PRETTY_FUNCTION__))
5469 "only valid in functions with gc specified, enforced by Verifier")((F->hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? static_cast<void> (0) : __assert_fail ("F->hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5469, __PRETTY_FUNCTION__))
;
5470 assert(GFI && "implied by previous")((GFI && "implied by previous") ? static_cast<void
> (0) : __assert_fail ("GFI && \"implied by previous\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5470, __PRETTY_FUNCTION__))
;
5471 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5472 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5473
5474 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5475 GFI->addStackRoot(FI->getIndex(), TypeMap);
5476 return nullptr;
5477 }
5478 case Intrinsic::gcread:
5479 case Intrinsic::gcwrite:
5480 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5480)
;
5481 case Intrinsic::flt_rounds:
5482 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5483 return nullptr;
5484
5485 case Intrinsic::expect: {
5486 // Just replace __builtin_expect(exp, c) with EXP.
5487 setValue(&I, getValue(I.getArgOperand(0)));
5488 return nullptr;
5489 }
5490
5491 case Intrinsic::debugtrap:
5492 case Intrinsic::trap: {
5493 StringRef TrapFuncName =
5494 I.getAttributes()
5495 .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
5496 .getValueAsString();
5497 if (TrapFuncName.empty()) {
5498 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5499 ISD::TRAP : ISD::DEBUGTRAP;
5500 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5501 return nullptr;
5502 }
5503 TargetLowering::ArgListTy Args;
5504
5505 TargetLowering::CallLoweringInfo CLI(DAG);
5506 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5507 CallingConv::C, I.getType(),
5508 DAG.getExternalSymbol(TrapFuncName.data(),
5509 TLI.getPointerTy(DAG.getDataLayout())),
5510 std::move(Args));
5511
5512 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5513 DAG.setRoot(Result.second);
5514 return nullptr;
5515 }
5516
5517 case Intrinsic::uadd_with_overflow:
5518 case Intrinsic::sadd_with_overflow:
5519 case Intrinsic::usub_with_overflow:
5520 case Intrinsic::ssub_with_overflow:
5521 case Intrinsic::umul_with_overflow:
5522 case Intrinsic::smul_with_overflow: {
5523 ISD::NodeType Op;
5524 switch (Intrinsic) {
5525 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5525)
; // Can't reach here.
5526 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5527 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5528 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5529 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5530 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5531 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5532 }
5533 SDValue Op1 = getValue(I.getArgOperand(0));
5534 SDValue Op2 = getValue(I.getArgOperand(1));
5535
5536 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5537 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5538 return nullptr;
5539 }
5540 case Intrinsic::prefetch: {
5541 SDValue Ops[5];
5542 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5543 Ops[0] = getRoot();
5544 Ops[1] = getValue(I.getArgOperand(0));
5545 Ops[2] = getValue(I.getArgOperand(1));
5546 Ops[3] = getValue(I.getArgOperand(2));
5547 Ops[4] = getValue(I.getArgOperand(3));
5548 DAG.setRoot(DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
5549 DAG.getVTList(MVT::Other), Ops,
5550 EVT::getIntegerVT(*Context, 8),
5551 MachinePointerInfo(I.getArgOperand(0)),
5552 0, /* align */
5553 false, /* volatile */
5554 rw==0, /* read */
5555 rw==1)); /* write */
5556 return nullptr;
5557 }
5558 case Intrinsic::lifetime_start:
5559 case Intrinsic::lifetime_end: {
5560 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
5561 // Stack coloring is not enabled in O0, discard region information.
5562 if (TM.getOptLevel() == CodeGenOpt::None)
5563 return nullptr;
5564
5565 SmallVector<Value *, 4> Allocas;
5566 GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
5567
5568 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
5569 E = Allocas.end(); Object != E; ++Object) {
5570 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
5571
5572 // Could not find an Alloca.
5573 if (!LifetimeObject)
5574 continue;
5575
5576 // First check that the Alloca is static, otherwise it won't have a
5577 // valid frame index.
5578 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
5579 if (SI == FuncInfo.StaticAllocaMap.end())
5580 return nullptr;
5581
5582 int FI = SI->second;
5583
5584 SDValue Ops[2];
5585 Ops[0] = getRoot();
5586 Ops[1] =
5587 DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true);
5588 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
5589
5590 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
5591 DAG.setRoot(Res);
5592 }
5593 return nullptr;
5594 }
5595 case Intrinsic::invariant_start:
5596 // Discard region information.
5597 setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
5598 return nullptr;
5599 case Intrinsic::invariant_end:
5600 // Discard region information.
5601 return nullptr;
5602 case Intrinsic::clear_cache:
5603 return TLI.getClearCacheBuiltinName();
5604 case Intrinsic::donothing:
5605 // ignore
5606 return nullptr;
5607 case Intrinsic::experimental_stackmap: {
5608 visitStackmap(I);
5609 return nullptr;
5610 }
5611 case Intrinsic::experimental_patchpoint_void:
5612 case Intrinsic::experimental_patchpoint_i64: {
5613 visitPatchpoint(&I);
5614 return nullptr;
5615 }
5616 case Intrinsic::experimental_gc_statepoint: {
5617 LowerStatepoint(ImmutableStatepoint(&I));
5618 return nullptr;
5619 }
5620 case Intrinsic::experimental_gc_result: {
5621 visitGCResult(cast<GCResultInst>(I));
5622 return nullptr;
5623 }
5624 case Intrinsic::experimental_gc_relocate: {
5625 visitGCRelocate(cast<GCRelocateInst>(I));
5626 return nullptr;
5627 }
5628 case Intrinsic::instrprof_increment:
5629 llvm_unreachable("instrprof failed to lower an increment")::llvm::llvm_unreachable_internal("instrprof failed to lower an increment"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5629)
;
5630 case Intrinsic::instrprof_value_profile:
5631 llvm_unreachable("instrprof failed to lower a value profiling call")::llvm::llvm_unreachable_internal("instrprof failed to lower a value profiling call"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5631)
;
5632 case Intrinsic::localescape: {
5633 MachineFunction &MF = DAG.getMachineFunction();
5634 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
5635
5636 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
5637 // is the same on all targets.
5638 for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
5639 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
5640 if (isa<ConstantPointerNull>(Arg))
5641 continue; // Skip null pointers. They represent a hole in index space.
5642 AllocaInst *Slot = cast<AllocaInst>(Arg);
5643 assert(FuncInfo.StaticAllocaMap.count(Slot) &&((FuncInfo.StaticAllocaMap.count(Slot) && "can only escape static allocas"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5644, __PRETTY_FUNCTION__))
5644 "can only escape static allocas")((FuncInfo.StaticAllocaMap.count(Slot) && "can only escape static allocas"
) ? static_cast<void> (0) : __assert_fail ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5644, __PRETTY_FUNCTION__))
;
5645 int FI = FuncInfo.StaticAllocaMap[Slot];
5646 MCSymbol *FrameAllocSym =
5647 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5648 GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
5649 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
5650 TII->get(TargetOpcode::LOCAL_ESCAPE))
5651 .addSym(FrameAllocSym)
5652 .addFrameIndex(FI);
5653 }
5654
5655 return nullptr;
5656 }
5657
5658 case Intrinsic::localrecover: {
5659 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
5660 MachineFunction &MF = DAG.getMachineFunction();
5661 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
5662
5663 // Get the symbol that defines the frame offset.
5664 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
5665 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
5666 unsigned IdxVal = unsigned(Idx->getLimitedValue(INT_MAX2147483647));
5667 MCSymbol *FrameAllocSym =
5668 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
5669 GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
5670
5671 // Create a MCSymbol for the label to avoid any target lowering
5672 // that would make this PC relative.
5673 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
5674 SDValue OffsetVal =
5675 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
5676
5677 // Add the offset to the FP.
5678 Value *FP = I.getArgOperand(1);
5679 SDValue FPVal = getValue(FP);
5680 SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
5681 setValue(&I, Add);
5682
5683 return nullptr;
5684 }
5685
5686 case Intrinsic::eh_exceptionpointer:
5687 case Intrinsic::eh_exceptioncode: {
5688 // Get the exception pointer vreg, copy from it, and resize it to fit.
5689 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
5690 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5691 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
5692 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
5693 SDValue N =
5694 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
5695 if (Intrinsic == Intrinsic::eh_exceptioncode)
5696 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
5697 setValue(&I, N);
5698 return nullptr;
5699 }
5700 case Intrinsic::xray_customevent: {
5701 // Here we want to make sure that the intrinsic behaves as if it has a
5702 // specific calling convention, and only for x86_64.
5703 // FIXME: Support other platforms later.
5704 const auto &Triple = DAG.getTarget().getTargetTriple();
5705 if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
5706 return nullptr;
5707
5708 SDLoc DL = getCurSDLoc();
5709 SmallVector<SDValue, 8> Ops;
5710
5711 // We want to say that we always want the arguments in registers.
5712 SDValue LogEntryVal = getValue(I.getArgOperand(0));
5713 SDValue StrSizeVal = getValue(I.getArgOperand(1));
5714 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
5715 SDValue Chain = getRoot();
5716 Ops.push_back(LogEntryVal);
5717 Ops.push_back(StrSizeVal);
5718 Ops.push_back(Chain);
5719
5720 // We need to enforce the calling convention for the callsite, so that
5721 // argument ordering is enforced correctly, and that register allocation can
5722 // see that some registers may be assumed clobbered and have to preserve
5723 // them across calls to the intrinsic.
5724 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
5725 DL, NodeTys, Ops);
5726 SDValue patchableNode = SDValue(MN, 0);
5727 DAG.setRoot(patchableNode);
5728 setValue(&I, patchableNode);
5729 return nullptr;
5730 }
5731 case Intrinsic::experimental_deoptimize:
5732 LowerDeoptimizeCall(&I);
5733 return nullptr;
5734
5735 case Intrinsic::experimental_vector_reduce_fadd:
5736 case Intrinsic::experimental_vector_reduce_fmul:
5737 case Intrinsic::experimental_vector_reduce_add:
5738 case Intrinsic::experimental_vector_reduce_mul:
5739 case Intrinsic::experimental_vector_reduce_and:
5740 case Intrinsic::experimental_vector_reduce_or:
5741 case Intrinsic::experimental_vector_reduce_xor:
5742 case Intrinsic::experimental_vector_reduce_smax:
5743 case Intrinsic::experimental_vector_reduce_smin:
5744 case Intrinsic::experimental_vector_reduce_umax:
5745 case Intrinsic::experimental_vector_reduce_umin:
5746 case Intrinsic::experimental_vector_reduce_fmax:
5747 case Intrinsic::experimental_vector_reduce_fmin: {
5748 visitVectorReduce(I, Intrinsic);
5749 return nullptr;
5750 }
5751
5752 }
5753}
5754
5755void SelectionDAGBuilder::visitConstrainedFPIntrinsic(const CallInst &I,
5756 unsigned Intrinsic) {
5757 SDLoc sdl = getCurSDLoc();
5758 unsigned Opcode;
5759 switch (Intrinsic) {
5760 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5760)
; // Can't reach here.
5761 case Intrinsic::experimental_constrained_fadd:
5762 Opcode = ISD::STRICT_FADD;
5763 break;
5764 case Intrinsic::experimental_constrained_fsub:
5765 Opcode = ISD::STRICT_FSUB;
5766 break;
5767 case Intrinsic::experimental_constrained_fmul:
5768 Opcode = ISD::STRICT_FMUL;
5769 break;
5770 case Intrinsic::experimental_constrained_fdiv:
5771 Opcode = ISD::STRICT_FDIV;
5772 break;
5773 case Intrinsic::experimental_constrained_frem:
5774 Opcode = ISD::STRICT_FREM;
5775 break;
5776 }
5777 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5778 SDValue Chain = getRoot();
5779 SDValue Ops[3] = { Chain, getValue(I.getArgOperand(0)),
5780 getValue(I.getArgOperand(1)) };
5781 SmallVector<EVT, 4> ValueVTs;
5782 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5783 ValueVTs.push_back(MVT::Other); // Out chain
5784
5785 SDVTList VTs = DAG.getVTList(ValueVTs);
5786 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Ops);
5787
5788 assert(Result.getNode()->getNumValues() == 2)((Result.getNode()->getNumValues() == 2) ? static_cast<
void> (0) : __assert_fail ("Result.getNode()->getNumValues() == 2"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5788, __PRETTY_FUNCTION__))
;
5789 SDValue OutChain = Result.getValue(1);
5790 DAG.setRoot(OutChain);
5791 SDValue FPResult = Result.getValue(0);
5792 setValue(&I, FPResult);
5793}
5794
5795std::pair<SDValue, SDValue>
5796SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
5797 const BasicBlock *EHPadBB) {
5798 MachineFunction &MF = DAG.getMachineFunction();
5799 MachineModuleInfo &MMI = MF.getMMI();
5800 MCSymbol *BeginLabel = nullptr;
5801
5802 if (EHPadBB) {
5803 // Insert a label before the invoke call to mark the try range. This can be
5804 // used to detect deletion of the invoke via the MachineModuleInfo.
5805 BeginLabel = MMI.getContext().createTempSymbol();
5806
5807 // For SjLj, keep track of which landing pads go with which invokes
5808 // so as to maintain the ordering of pads in the LSDA.
5809 unsigned CallSiteIndex = MMI.getCurrentCallSite();
5810 if (CallSiteIndex) {
5811 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
5812 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
5813
5814 // Now that the call site is handled, stop tracking it.
5815 MMI.setCurrentCallSite(0);
5816 }
5817
5818 // Both PendingLoads and PendingExports must be flushed here;
5819 // this call might not return.
5820 (void)getRoot();
5821 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
5822
5823 CLI.setChain(getRoot());
5824 }
5825 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5826 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5827
5828 assert((CLI.IsTailCall || Result.second.getNode()) &&(((CLI.IsTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"
) ? static_cast<void> (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5829, __PRETTY_FUNCTION__))
5829 "Non-null chain expected with non-tail call!")(((CLI.IsTailCall || Result.second.getNode()) && "Non-null chain expected with non-tail call!"
) ? static_cast<void> (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5829, __PRETTY_FUNCTION__))
;
5830 assert((Result.second.getNode() || !Result.first.getNode()) &&(((Result.second.getNode() || !Result.first.getNode()) &&
"Null value expected with tail call!") ? static_cast<void
> (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5831, __PRETTY_FUNCTION__))
5831 "Null value expected with tail call!")(((Result.second.getNode() || !Result.first.getNode()) &&
"Null value expected with tail call!") ? static_cast<void
> (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5831, __PRETTY_FUNCTION__))
;
5832
5833 if (!Result.second.getNode()) {
5834 // As a special case, a null chain means that a tail call has been emitted
5835 // and the DAG root is already updated.
5836 HasTailCall = true;
5837
5838 // Since there's no actual continuation from this block, nothing can be
5839 // relying on us setting vregs for them.
5840 PendingExports.clear();
5841 } else {
5842 DAG.setRoot(Result.second);
5843 }
5844
5845 if (EHPadBB) {
5846 // Insert a label at the end of the invoke call to mark the try range. This
5847 // can be used to detect deletion of the invoke via the MachineModuleInfo.
5848 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
5849 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
5850
5851 // Inform MachineModuleInfo of range.
5852 if (MF.hasEHFunclets()) {
5853 assert(CLI.CS)((CLI.CS) ? static_cast<void> (0) : __assert_fail ("CLI.CS"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5853, __PRETTY_FUNCTION__))
;
5854 WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
5855 EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS->getInstruction()),
5856 BeginLabel, EndLabel);
5857 } else {
5858 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
5859 }
5860 }
5861
5862 return Result;
5863}
5864
5865void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
5866 bool isTailCall,
5867 const BasicBlock *EHPadBB) {
5868 auto &DL = DAG.getDataLayout();
5869 FunctionType *FTy = CS.getFunctionType();
5870 Type *RetTy = CS.getType();
5871
5872 TargetLowering::ArgListTy Args;
5873 Args.reserve(CS.arg_size());
5874
5875 const Value *SwiftErrorVal = nullptr;
5876 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5877
5878 // We can't tail call inside a function with a swifterror argument. Lowering
5879 // does not support this yet. It would have to move into the swifterror
5880 // register before the call.
5881 auto *Caller = CS.getInstruction()->getParent()->getParent();
5882 if (TLI.supportSwiftError() &&
5883 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
5884 isTailCall = false;
5885
5886 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
5887 i != e; ++i) {
5888 TargetLowering::ArgListEntry Entry;
5889 const Value *V = *i;
5890
5891 // Skip empty types
5892 if (V->getType()->isEmptyTy())
5893 continue;
5894
5895 SDValue ArgNode = getValue(V);
5896 Entry.Node = ArgNode; Entry.Ty = V->getType();
5897
5898 Entry.setAttributes(&CS, i - CS.arg_begin());
5899
5900 // Use swifterror virtual register as input to the call.
5901 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
5902 SwiftErrorVal = V;
5903 // We find the virtual register for the actual swifterror argument.
5904 // Instead of using the Value, we use the virtual register instead.
5905 Entry.Node =
5906 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVReg(FuncInfo.MBB, V),
5907 EVT(TLI.getPointerTy(DL)));
5908 }
5909
5910 Args.push_back(Entry);
5911
5912 // If we have an explicit sret argument that is an Instruction, (i.e., it
5913 // might point to function-local memory), we can't meaningfully tail-call.
5914 if (Entry.IsSRet && isa<Instruction>(V))
5915 isTailCall = false;
5916 }
5917
5918 // Check if target-independent constraints permit a tail call here.
5919 // Target-dependent constraints are checked within TLI->LowerCallTo.
5920 if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
5921 isTailCall = false;
5922
5923 // Disable tail calls if there is an swifterror argument. Targets have not
5924 // been updated to support tail calls.
5925 if (TLI.supportSwiftError() && SwiftErrorVal)
5926 isTailCall = false;
5927
5928 TargetLowering::CallLoweringInfo CLI(DAG);
5929 CLI.setDebugLoc(getCurSDLoc())
5930 .setChain(getRoot())
5931 .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
5932 .setTailCall(isTailCall)
5933 .setConvergent(CS.isConvergent());
5934 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
5935
5936 if (Result.first.getNode()) {
5937 const Instruction *Inst = CS.getInstruction();
5938 Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
5939 setValue(Inst, Result.first);
5940 }
5941
5942 // The last element of CLI.InVals has the SDValue for swifterror return.
5943 // Here we copy it to a virtual register and update SwiftErrorMap for
5944 // book-keeping.
5945 if (SwiftErrorVal && TLI.supportSwiftError()) {
5946 // Get the last element of InVals.
5947 SDValue Src = CLI.InVals.back();
5948 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
5949 unsigned VReg = FuncInfo.MF->getRegInfo().createVirtualRegister(RC);
5950 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
5951 // We update the virtual register for the actual swifterror argument.
5952 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
5953 DAG.setRoot(CopyNode);
5954 }
5955}
5956
5957/// Return true if it only matters that the value is equal or not-equal to zero.
5958static bool IsOnlyUsedInZeroEqualityComparison(const Value *V) {
5959 for (const User *U : V->users()) {
5960 if (const ICmpInst *IC = dyn_cast<ICmpInst>(U))
5961 if (IC->isEquality())
5962 if (const Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
5963 if (C->isNullValue())
5964 continue;
5965 // Unknown instruction.
5966 return false;
5967 }
5968 return true;
5969}
5970
5971static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
5972 SelectionDAGBuilder &Builder) {
5973
5974 // Check to see if this load can be trivially constant folded, e.g. if the
5975 // input is from a string literal.
5976 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
5977 // Cast pointer to the type we really want to load.
5978 Type *LoadTy =
5979 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
5980 if (LoadVT.isVector())
5981 LoadTy = VectorType::get(LoadTy, LoadVT.getVectorNumElements());
5982
5983 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
5984 PointerType::getUnqual(LoadTy));
5985
5986 if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
5987 const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
5988 return Builder.getValue(LoadCst);
5989 }
5990
5991 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
5992 // still constant memory, the input chain can be the entry node.
5993 SDValue Root;
5994 bool ConstantMemory = false;
5995
5996 // Do not serialize (non-volatile) loads of constant memory with anything.
5997 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
5998 Root = Builder.DAG.getEntryNode();
5999 ConstantMemory = true;
6000 } else {
6001 // Do not serialize non-volatile loads against each other.
6002 Root = Builder.DAG.getRoot();
6003 }
6004
6005 SDValue Ptr = Builder.getValue(PtrVal);
6006 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
6007 Ptr, MachinePointerInfo(PtrVal),
6008 /* Alignment = */ 1);
6009
6010 if (!ConstantMemory)
6011 Builder.PendingLoads.push_back(LoadVal.getValue(1));
6012 return LoadVal;
6013}
6014
6015/// Record the value for an instruction that produces an integer result,
6016/// converting the type where necessary.
6017void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
6018 SDValue Value,
6019 bool IsSigned) {
6020 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6021 I.getType(), true);
6022 if (IsSigned)
6023 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
6024 else
6025 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
6026 setValue(&I, Value);
6027}
6028
6029/// See if we can lower a memcmp call into an optimized form. If so, return
6030/// true and lower it. Otherwise return false, and it will be lowered like a
6031/// normal call.
6032/// The caller already checked that \p I calls the appropriate LibFunc with a
6033/// correct prototype.
6034bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6035 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6036 const Value *Size = I.getArgOperand(2);
6037 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6038 if (CSize && CSize->getZExtValue() == 0) {
6039 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6040 I.getType(), true);
6041 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6042 return true;
6043 }
6044
6045 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6046 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
6047 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
6048 getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
6049 if (Res.first.getNode()) {
6050 processIntegerCallValue(I, Res.first, true);
6051 PendingLoads.push_back(Res.second);
6052 return true;
6053 }
6054
6055 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
6056 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
6057 if (!CSize || !IsOnlyUsedInZeroEqualityComparison(&I))
6058 return false;
6059
6060 // If the target has a fast compare for the given size, it will return a
6061 // preferred load type for that size. Require that the load VT is legal and
6062 // that the target supports unaligned loads of that type. Otherwise, return
6063 // INVALID.
6064 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
6065 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6066 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
6067 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
6068 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6069 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6070 // TODO: Check alignment of src and dest ptrs.
6071 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6072 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6073 if (!TLI.isTypeLegal(LVT) ||
6074 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
6075 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
6076 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
6077 }
6078
6079 return LVT;
6080 };
6081
6082 // This turns into unaligned loads. We only do this if the target natively
6083 // supports the MVT we'll be loading or if it is small enough (<= 4) that
6084 // we'll only produce a small number of byte loads.
6085 MVT LoadVT;
6086 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
6087 switch (NumBitsToCompare) {
6088 default:
6089 return false;
6090 case 16:
6091 LoadVT = MVT::i16;
6092 break;
6093 case 32:
6094 LoadVT = MVT::i32;
6095 break;
6096 case 64:
6097 case 128:
6098 case 256:
6099 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
6100 break;
6101 }
6102
6103 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
6104 return false;
6105
6106 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
6107 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
6108
6109 // Bitcast to a wide integer type if the loads are vectors.
6110 if (LoadVT.isVector()) {
6111 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
6112 LoadL = DAG.getBitcast(CmpVT, LoadL);
6113 LoadR = DAG.getBitcast(CmpVT, LoadR);
6114 }
6115
6116 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
6117 processIntegerCallValue(I, Cmp, false);
6118 return true;
6119}
6120
6121/// See if we can lower a memchr call into an optimized form. If so, return
6122/// true and lower it. Otherwise return false, and it will be lowered like a
6123/// normal call.
6124/// The caller already checked that \p I calls the appropriate LibFunc with a
6125/// correct prototype.
6126bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6127 const Value *Src = I.getArgOperand(0);
6128 const Value *Char = I.getArgOperand(1);
6129 const Value *Length = I.getArgOperand(2);
6130
6131 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6132 std::pair<SDValue, SDValue> Res =
6133 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6134 getValue(Src), getValue(Char), getValue(Length),
6135 MachinePointerInfo(Src));
6136 if (Res.first.getNode()) {
6137 setValue(&I, Res.first);
6138 PendingLoads.push_back(Res.second);
6139 return true;
6140 }
6141
6142 return false;
6143}
6144
6145/// See if we can lower a mempcpy call into an optimized form. If so, return
6146/// true and lower it. Otherwise return false, and it will be lowered like a
6147/// normal call.
6148/// The caller already checked that \p I calls the appropriate LibFunc with a
6149/// correct prototype.
6150bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6151 SDValue Dst = getValue(I.getArgOperand(0));
6152 SDValue Src = getValue(I.getArgOperand(1));
6153 SDValue Size = getValue(I.getArgOperand(2));
6154
6155 unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6156 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6157 unsigned Align = std::min(DstAlign, SrcAlign);
6158 if (Align == 0) // Alignment of one or both could not be inferred.
6159 Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6160
6161 bool isVol = false;
6162 SDLoc sdl = getCurSDLoc();
6163
6164 // In the mempcpy context we need to pass in a false value for isTailCall
6165 // because the return pointer needs to be adjusted by the size of
6166 // the copied memory.
6167 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6168 false, /*isTailCall=*/false,
6169 MachinePointerInfo(I.getArgOperand(0)),
6170 MachinePointerInfo(I.getArgOperand(1)));
6171 assert(MC.getNode() != nullptr &&((MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **"
) ? static_cast<void> (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6172, __PRETTY_FUNCTION__))
6172 "** memcpy should not be lowered as TailCall in mempcpy context **")((MC.getNode() != nullptr && "** memcpy should not be lowered as TailCall in mempcpy context **"
) ? static_cast<void> (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6172, __PRETTY_FUNCTION__))
;
6173 DAG.setRoot(MC);
6174
6175 // Check if Size needs to be truncated or extended.
6176 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6177
6178 // Adjust return pointer to point just past the last dst byte.
6179 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6180 Dst, Size);
6181 setValue(&I, DstPlusSize);
6182 return true;
6183}
6184
6185/// See if we can lower a strcpy call into an optimized form. If so, return
6186/// true and lower it, otherwise return false and it will be lowered like a
6187/// normal call.
6188/// The caller already checked that \p I calls the appropriate LibFunc with a
6189/// correct prototype.
6190bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6191 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6192
6193 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6194 std::pair<SDValue, SDValue> Res =
6195 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6196 getValue(Arg0), getValue(Arg1),
6197 MachinePointerInfo(Arg0),
6198 MachinePointerInfo(Arg1), isStpcpy);
6199 if (Res.first.getNode()) {
6200 setValue(&I, Res.first);
6201 DAG.setRoot(Res.second);
6202 return true;
6203 }
6204
6205 return false;
6206}
6207
6208/// See if we can lower a strcmp call into an optimized form. If so, return
6209/// true and lower it, otherwise return false and it will be lowered like a
6210/// normal call.
6211/// The caller already checked that \p I calls the appropriate LibFunc with a
6212/// correct prototype.
6213bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6214 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6215
6216 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6217 std::pair<SDValue, SDValue> Res =
6218 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6219 getValue(Arg0), getValue(Arg1),
6220 MachinePointerInfo(Arg0),
6221 MachinePointerInfo(Arg1));
6222 if (Res.first.getNode()) {
6223 processIntegerCallValue(I, Res.first, true);
6224 PendingLoads.push_back(Res.second);
6225 return true;
6226 }
6227
6228 return false;
6229}
6230
6231/// See if we can lower a strlen call into an optimized form. If so, return
6232/// true and lower it, otherwise return false and it will be lowered like a
6233/// normal call.
6234/// The caller already checked that \p I calls the appropriate LibFunc with a
6235/// correct prototype.
6236bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6237 const Value *Arg0 = I.getArgOperand(0);
6238
6239 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6240 std::pair<SDValue, SDValue> Res =
6241 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6242 getValue(Arg0), MachinePointerInfo(Arg0));
6243 if (Res.first.getNode()) {
6244 processIntegerCallValue(I, Res.first, false);
6245 PendingLoads.push_back(Res.second);
6246 return true;
6247 }
6248
6249 return false;
6250}
6251
6252/// See if we can lower a strnlen call into an optimized form. If so, return
6253/// true and lower it, otherwise return false and it will be lowered like a
6254/// normal call.
6255/// The caller already checked that \p I calls the appropriate LibFunc with a
6256/// correct prototype.
6257bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6258 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6259
6260 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6261 std::pair<SDValue, SDValue> Res =
6262 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6263 getValue(Arg0), getValue(Arg1),
6264 MachinePointerInfo(Arg0));
6265 if (Res.first.getNode()) {
6266 processIntegerCallValue(I, Res.first, false);
6267 PendingLoads.push_back(Res.second);
6268 return true;
6269 }
6270
6271 return false;
6272}
6273
6274/// See if we can lower a unary floating-point operation into an SDNode with
6275/// the specified Opcode. If so, return true and lower it, otherwise return
6276/// false and it will be lowered like a normal call.
6277/// The caller already checked that \p I calls the appropriate LibFunc with a
6278/// correct prototype.
6279bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6280 unsigned Opcode) {
6281 // We already checked this call's prototype; verify it doesn't modify errno.
6282 if (!I.onlyReadsMemory())
6283 return false;
6284
6285 SDValue Tmp = getValue(I.getArgOperand(0));
6286 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6287 return true;
6288}
6289
6290/// See if we can lower a binary floating-point operation into an SDNode with
6291/// the specified Opcode. If so, return true and lower it. Otherwise return
6292/// false, and it will be lowered like a normal call.
6293/// The caller already checked that \p I calls the appropriate LibFunc with a
6294/// correct prototype.
6295bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6296 unsigned Opcode) {
6297 // We already checked this call's prototype; verify it doesn't modify errno.
6298 if (!I.onlyReadsMemory())
6299 return false;
6300
6301 SDValue Tmp0 = getValue(I.getArgOperand(0));
6302 SDValue Tmp1 = getValue(I.getArgOperand(1));
6303 EVT VT = Tmp0.getValueType();
6304 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6305 return true;
6306}
6307
6308void SelectionDAGBuilder::visitCall(const CallInst &I) {
6309 // Handle inline assembly differently.
6310 if (isa<InlineAsm>(I.getCalledValue())) {
6311 visitInlineAsm(&I);
6312 return;
6313 }
6314
6315 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6316 computeUsesVAFloatArgument(I, MMI);
6317
6318 const char *RenameFn = nullptr;
6319 if (Function *F = I.getCalledFunction()) {
6320 if (F->isDeclaration()) {
6321 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) {
6322 if (unsigned IID = II->getIntrinsicID(F)) {
6323 RenameFn = visitIntrinsicCall(I, IID);
6324 if (!RenameFn)
6325 return;
6326 }
6327 }
6328 if (Intrinsic::ID IID = F->getIntrinsicID()) {
6329 RenameFn = visitIntrinsicCall(I, IID);
6330 if (!RenameFn)
6331 return;
6332 }
6333 }
6334
6335 // Check for well-known libc/libm calls. If the function is internal, it
6336 // can't be a library call. Don't do the check if marked as nobuiltin for
6337 // some reason.
6338 LibFunc Func;
6339 if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() &&
6340 LibInfo->getLibFunc(*F, Func) &&
6341 LibInfo->hasOptimizedCodeGen(Func)) {
6342 switch (Func) {
6343 default: break;
6344 case LibFunc_copysign:
6345 case LibFunc_copysignf:
6346 case LibFunc_copysignl:
6347 // We already checked this call's prototype; verify it doesn't modify
6348 // errno.
6349 if (I.onlyReadsMemory()) {
6350 SDValue LHS = getValue(I.getArgOperand(0));
6351 SDValue RHS = getValue(I.getArgOperand(1));
6352 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
6353 LHS.getValueType(), LHS, RHS));
6354 return;
6355 }
6356 break;
6357 case LibFunc_fabs:
6358 case LibFunc_fabsf:
6359 case LibFunc_fabsl:
6360 if (visitUnaryFloatCall(I, ISD::FABS))
6361 return;
6362 break;
6363 case LibFunc_fmin:
6364 case LibFunc_fminf:
6365 case LibFunc_fminl:
6366 if (visitBinaryFloatCall(I, ISD::FMINNUM))
6367 return;
6368 break;
6369 case LibFunc_fmax:
6370 case LibFunc_fmaxf:
6371 case LibFunc_fmaxl:
6372 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
6373 return;
6374 break;
6375 case LibFunc_sin:
6376 case LibFunc_sinf:
6377 case LibFunc_sinl:
6378 if (visitUnaryFloatCall(I, ISD::FSIN))
6379 return;
6380 break;
6381 case LibFunc_cos:
6382 case LibFunc_cosf:
6383 case LibFunc_cosl:
6384 if (visitUnaryFloatCall(I, ISD::FCOS))
6385 return;
6386 break;
6387 case LibFunc_sqrt:
6388 case LibFunc_sqrtf:
6389 case LibFunc_sqrtl:
6390 case LibFunc_sqrt_finite:
6391 case LibFunc_sqrtf_finite:
6392 case LibFunc_sqrtl_finite:
6393 if (visitUnaryFloatCall(I, ISD::FSQRT))
6394 return;
6395 break;
6396 case LibFunc_floor:
6397 case LibFunc_floorf:
6398 case LibFunc_floorl:
6399 if (visitUnaryFloatCall(I, ISD::FFLOOR))
6400 return;
6401 break;
6402 case LibFunc_nearbyint:
6403 case LibFunc_nearbyintf:
6404 case LibFunc_nearbyintl:
6405 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
6406 return;
6407 break;
6408 case LibFunc_ceil:
6409 case LibFunc_ceilf:
6410 case LibFunc_ceill:
6411 if (visitUnaryFloatCall(I, ISD::FCEIL))
6412 return;
6413 break;
6414 case LibFunc_rint:
6415 case LibFunc_rintf:
6416 case LibFunc_rintl:
6417 if (visitUnaryFloatCall(I, ISD::FRINT))
6418 return;
6419 break;
6420 case LibFunc_round:
6421 case LibFunc_roundf:
6422 case LibFunc_roundl:
6423 if (visitUnaryFloatCall(I, ISD::FROUND))
6424 return;
6425 break;
6426 case LibFunc_trunc:
6427 case LibFunc_truncf:
6428 case LibFunc_truncl:
6429 if (visitUnaryFloatCall(I, ISD::FTRUNC))
6430 return;
6431 break;
6432 case LibFunc_log2:
6433 case LibFunc_log2f:
6434 case LibFunc_log2l:
6435 if (visitUnaryFloatCall(I, ISD::FLOG2))
6436 return;
6437 break;
6438 case LibFunc_exp2:
6439 case LibFunc_exp2f:
6440 case LibFunc_exp2l:
6441 if (visitUnaryFloatCall(I, ISD::FEXP2))
6442 return;
6443 break;
6444 case LibFunc_memcmp:
6445 if (visitMemCmpCall(I))
6446 return;
6447 break;
6448 case LibFunc_mempcpy:
6449 if (visitMemPCpyCall(I))
6450 return;
6451 break;
6452 case LibFunc_memchr:
6453 if (visitMemChrCall(I))
6454 return;
6455 break;
6456 case LibFunc_strcpy:
6457 if (visitStrCpyCall(I, false))
6458 return;
6459 break;
6460 case LibFunc_stpcpy:
6461 if (visitStrCpyCall(I, true))
6462 return;
6463 break;
6464 case LibFunc_strcmp:
6465 if (visitStrCmpCall(I))
6466 return;
6467 break;
6468 case LibFunc_strlen:
6469 if (visitStrLenCall(I))
6470 return;
6471 break;
6472 case LibFunc_strnlen:
6473 if (visitStrNLenCall(I))
6474 return;
6475 break;
6476 }
6477 }
6478 }
6479
6480 SDValue Callee;
6481 if (!RenameFn)
6482 Callee = getValue(I.getCalledValue());
6483 else
6484 Callee = DAG.getExternalSymbol(
6485 RenameFn,
6486 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
6487
6488 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
6489 // have to do anything here to lower funclet bundles.
6490 assert(!I.hasOperandBundlesOtherThan(((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6492, __PRETTY_FUNCTION__))
6491 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6492, __PRETTY_FUNCTION__))
6492 "Cannot lower calls with arbitrary operand bundles!")((!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext
::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? static_cast<void> (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6492, __PRETTY_FUNCTION__))
;
6493
6494 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
6495 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
6496 else
6497 // Check if we can potentially perform a tail call. More detailed checking
6498 // is be done within LowerCallTo, after more information about the call is
6499 // known.
6500 LowerCallTo(&I, Callee, I.isTailCall());
6501}
6502
6503namespace {
6504
6505/// AsmOperandInfo - This contains information for each constraint that we are
6506/// lowering.
6507class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
6508public:
6509 /// CallOperand - If this is the result output operand or a clobber
6510 /// this is null, otherwise it is the incoming operand to the CallInst.
6511 /// This gets modified as the asm is processed.
6512 SDValue CallOperand;
6513
6514 /// AssignedRegs - If this is a register or register class operand, this
6515 /// contains the set of register corresponding to the operand.
6516 RegsForValue AssignedRegs;
6517
6518 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
6519 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr,0) {
6520 }
6521
6522 /// Whether or not this operand accesses memory
6523 bool hasMemory(const TargetLowering &TLI) const {
6524 // Indirect operand accesses access memory.
6525 if (isIndirect)
6526 return true;
6527
6528 for (const auto &Code : Codes)
6529 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
6530 return true;
6531
6532 return false;
6533 }
6534
6535 /// getCallOperandValEVT - Return the EVT of the Value* that this operand
6536 /// corresponds to. If there is no Value* for this operand, it returns
6537 /// MVT::Other.
6538 EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
6539 const DataLayout &DL) const {
6540 if (!CallOperandVal) return MVT::Other;
6541
6542 if (isa<BasicBlock>(CallOperandVal))
6543 return TLI.getPointerTy(DL);
6544
6545 llvm::Type *OpTy = CallOperandVal->getType();
6546
6547 // FIXME: code duplicated from TargetLowering::ParseConstraints().
6548 // If this is an indirect operand, the operand is a pointer to the
6549 // accessed type.
6550 if (isIndirect) {
6551 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
6552 if (!PtrTy)
6553 report_fatal_error("Indirect operand for inline asm not a pointer!");
6554 OpTy = PtrTy->getElementType();
6555 }
6556
6557 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6558 if (StructType *STy = dyn_cast<StructType>(OpTy))
6559 if (STy->getNumElements() == 1)
6560 OpTy = STy->getElementType(0);
6561
6562 // If OpTy is not a single value, it may be a struct/union that we
6563 // can tile with integers.
6564 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6565 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
6566 switch (BitSize) {
6567 default: break;
6568 case 1:
6569 case 8:
6570 case 16:
6571 case 32:
6572 case 64:
6573 case 128:
6574 OpTy = IntegerType::get(Context, BitSize);
6575 break;
6576 }
6577 }
6578
6579 return TLI.getValueType(DL, OpTy, true);
6580 }
6581};
6582
6583typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector;
6584
6585} // end anonymous namespace
6586
6587/// Make sure that the output operand \p OpInfo and its corresponding input
6588/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
6589/// out).
6590static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
6591 SDISelAsmOperandInfo &MatchingOpInfo,
6592 SelectionDAG &DAG) {
6593 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
6594 return;
6595
6596 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
6597 const auto &TLI = DAG.getTargetLoweringInfo();
6598
6599 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6600 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
6601 OpInfo.ConstraintVT);
6602 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6603 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
6604 MatchingOpInfo.ConstraintVT);
6605 if ((OpInfo.ConstraintVT.isInteger() !=
6606 MatchingOpInfo.ConstraintVT.isInteger()) ||
6607 (MatchRC.second != InputRC.second)) {
6608 // FIXME: error out in a more elegant fashion
6609 report_fatal_error("Unsupported asm: input constraint"
6610 " with a matching output constraint of"
6611 " incompatible type!");
6612 }
6613 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
6614}
6615
6616/// Get a direct memory input to behave well as an indirect operand.
6617/// This may introduce stores, hence the need for a \p Chain.
6618/// \return The (possibly updated) chain.
6619static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
6620 SDISelAsmOperandInfo &OpInfo,
6621 SelectionDAG &DAG) {
6622 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6623
6624 // If we don't have an indirect input, put it in the constpool if we can,
6625 // otherwise spill it to a stack slot.
6626 // TODO: This isn't quite right. We need to handle these according to
6627 // the addressing mode that the constraint wants. Also, this may take
6628 // an additional register for the computation and we don't want that
6629 // either.
6630
6631 // If the operand is a float, integer, or vector constant, spill to a
6632 // constant pool entry to get its address.
6633 const Value *OpVal = OpInfo.CallOperandVal;
6634 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
6635 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
6636 OpInfo.CallOperand = DAG.getConstantPool(
6637 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
6638 return Chain;
6639 }
6640
6641 // Otherwise, create a stack slot and emit a store to it before the asm.
6642 Type *Ty = OpVal->getType();
6643 auto &DL = DAG.getDataLayout();
6644 uint64_t TySize = DL.getTypeAllocSize(Ty);
6645 unsigned Align = DL.getPrefTypeAlignment(Ty);
6646 MachineFunction &MF = DAG.getMachineFunction();
6647 int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
6648 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
6649 Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
6650 MachinePointerInfo::getFixedStack(MF, SSFI));
6651 OpInfo.CallOperand = StackSlot;
6652
6653 return Chain;
6654}
6655
6656/// GetRegistersForValue - Assign registers (virtual or physical) for the
6657/// specified operand. We prefer to assign virtual registers, to allow the
6658/// register allocator to handle the assignment process. However, if the asm
6659/// uses features that we can't model on machineinstrs, we have SDISel do the
6660/// allocation. This produces generally horrible, but correct, code.
6661///
6662/// OpInfo describes the operand.
6663///
6664static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
6665 const SDLoc &DL,
6666 SDISelAsmOperandInfo &OpInfo) {
6667 LLVMContext &Context = *DAG.getContext();
6668
6669 MachineFunction &MF = DAG.getMachineFunction();
6670 SmallVector<unsigned, 4> Regs;
6671 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
6672
6673 // If this is a constraint for a single physreg, or a constraint for a
6674 // register class, find it.
6675 std::pair<unsigned, const TargetRegisterClass *> PhysReg =
6676 TLI.getRegForInlineAsmConstraint(&TRI, OpInfo.ConstraintCode,
1
Value assigned to 'PhysReg.second'
6677 OpInfo.ConstraintVT);
6678
6679 unsigned NumRegs = 1;
6680 if (OpInfo.ConstraintVT != MVT::Other) {
2
Taking true branch
6681 // If this is a FP input in an integer register (or visa versa) insert a bit
6682 // cast of the input value. More generally, handle any case where the input
6683 // value disagrees with the register class we plan to stick this in.
6684 if (OpInfo.Type == InlineAsm::isInput && PhysReg.second &&
3
Assuming the condition is true
4
Assuming pointer value is null
5
Assuming the condition is false
6685 !TRI.isTypeLegalForClass(*PhysReg.second, OpInfo.ConstraintVT)) {
6686 // Try to convert to the first EVT that the reg class contains. If the
6687 // types are identical size, use a bitcast to convert (e.g. two differing
6688 // vector types).
6689 MVT RegVT = *TRI.legalclasstypes_begin(*PhysReg.second);
6690 if (RegVT.getSizeInBits() == OpInfo.CallOperand.getValueSizeInBits()) {
6691 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6692 RegVT, OpInfo.CallOperand);
6693 OpInfo.ConstraintVT = RegVT;
6694 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
6695 // If the input is a FP value and we want it in FP registers, do a
6696 // bitcast to the corresponding integer type. This turns an f64 value
6697 // into i64, which can be passed with two i32 values on a 32-bit
6698 // machine.
6699 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
6700 OpInfo.CallOperand = DAG.getNode(ISD::BITCAST, DL,
6701 RegVT, OpInfo.CallOperand);
6702 OpInfo.ConstraintVT = RegVT;
6703 }
6704 }
6705
6706 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
6707 }
6708
6709 MVT RegVT;
6710 EVT ValueVT = OpInfo.ConstraintVT;
6711
6712 // If this is a constraint for a specific physical register, like {r17},
6713 // assign it now.
6714 if (unsigned AssignedReg = PhysReg.first) {
6
Assuming 'AssignedReg' is not equal to 0
7
Taking true branch
6715 const TargetRegisterClass *RC = PhysReg.second;
8
'RC' initialized to a null pointer value
6716 if (OpInfo.ConstraintVT == MVT::Other)
9
Taking false branch
6717 ValueVT = *TRI.legalclasstypes_begin(*RC);
6718
6719 // Get the actual register value type. This is important, because the user
6720 // may have asked for (e.g.) the AX register in i32 type. We need to
6721 // remember that AX is actually i16 to get the right extension.
6722 RegVT = *TRI.legalclasstypes_begin(*RC);
10
Forming reference to null pointer
6723
6724 // This is a explicit reference to a physical register.
6725 Regs.push_back(AssignedReg);
6726
6727 // If this is an expanded reference, add the rest of the regs to Regs.
6728 if (NumRegs != 1) {
6729 TargetRegisterClass::iterator I = RC->begin();
6730 for (; *I != AssignedReg; ++I)
6731 assert(I != RC->end() && "Didn't find reg!")((I != RC->end() && "Didn't find reg!") ? static_cast
<void> (0) : __assert_fail ("I != RC->end() && \"Didn't find reg!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6731, __PRETTY_FUNCTION__))
;
6732
6733 // Already added the first reg.
6734 --NumRegs; ++I;
6735 for (; NumRegs; --NumRegs, ++I) {
6736 assert(I != RC->end() && "Ran out of registers to allocate!")((I != RC->end() && "Ran out of registers to allocate!"
) ? static_cast<void> (0) : __assert_fail ("I != RC->end() && \"Ran out of registers to allocate!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6736, __PRETTY_FUNCTION__))
;
6737 Regs.push_back(*I);
6738 }
6739 }
6740
6741 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6742 return;
6743 }
6744
6745 // Otherwise, if this was a reference to an LLVM register class, create vregs
6746 // for this reference.
6747 if (const TargetRegisterClass *RC = PhysReg.second) {
6748 RegVT = *TRI.legalclasstypes_begin(*RC);
6749 if (OpInfo.ConstraintVT == MVT::Other)
6750 ValueVT = RegVT;
6751
6752 // Create the appropriate number of virtual registers.
6753 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6754 for (; NumRegs; --NumRegs)
6755 Regs.push_back(RegInfo.createVirtualRegister(RC));
6756
6757 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
6758 return;
6759 }
6760
6761 // Otherwise, we couldn't allocate enough registers for this.
6762}
6763
6764static unsigned
6765findMatchingInlineAsmOperand(unsigned OperandNo,
6766 const std::vector<SDValue> &AsmNodeOperands) {
6767 // Scan until we find the definition we already emitted of this operand.
6768 unsigned CurOp = InlineAsm::Op_FirstOperand;
6769 for (; OperandNo; --OperandNo) {
6770 // Advance to the next operand.
6771 unsigned OpFlag =
6772 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
6773 assert((InlineAsm::isRegDefKind(OpFlag) ||(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6776, __PRETTY_FUNCTION__))
6774 InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6776, __PRETTY_FUNCTION__))
6775 InlineAsm::isMemKind(OpFlag)) &&(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6776, __PRETTY_FUNCTION__))
6776 "Skipped past definitions?")(((InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind
(OpFlag) || InlineAsm::isMemKind(OpFlag)) && "Skipped past definitions?"
) ? static_cast<void> (0) : __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6776, __PRETTY_FUNCTION__))
;
6777 CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
6778 }
6779 return CurOp;
6780}
6781
6782/// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
6783/// \return true if it has succeeded, false otherwise
6784static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
6785 MVT RegVT, SelectionDAG &DAG) {
6786 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6787 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
6788 for (unsigned i = 0, e = NumRegs; i != e; ++i) {
6789 if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
6790 Regs.push_back(RegInfo.createVirtualRegister(RC));
6791 else
6792 return false;
6793 }
6794 return true;
6795}
6796
6797class ExtraFlags {
6798 unsigned Flags = 0;
6799
6800public:
6801 explicit ExtraFlags(ImmutableCallSite CS) {
6802 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6803 if (IA->hasSideEffects())
6804 Flags |= InlineAsm::Extra_HasSideEffects;
6805 if (IA->isAlignStack())
6806 Flags |= InlineAsm::Extra_IsAlignStack;
6807 if (CS.isConvergent())
6808 Flags |= InlineAsm::Extra_IsConvergent;
6809 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
6810 }
6811
6812 void update(const llvm::TargetLowering::AsmOperandInfo &OpInfo) {
6813 // Ideally, we would only check against memory constraints. However, the
6814 // meaning of an Other constraint can be target-specific and we can't easily
6815 // reason about it. Therefore, be conservative and set MayLoad/MayStore
6816 // for Other constraints as well.
6817 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
6818 OpInfo.ConstraintType == TargetLowering::C_Other) {
6819 if (OpInfo.Type == InlineAsm::isInput)
6820 Flags |= InlineAsm::Extra_MayLoad;
6821 else if (OpInfo.Type == InlineAsm::isOutput)
6822 Flags |= InlineAsm::Extra_MayStore;
6823 else if (OpInfo.Type == InlineAsm::isClobber)
6824 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
6825 }
6826 }
6827
6828 unsigned get() const { return Flags; }
6829};
6830
6831/// visitInlineAsm - Handle a call to an InlineAsm object.
6832///
6833void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
6834 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
6835
6836 /// ConstraintOperands - Information about all of the constraints.
6837 SDISelAsmOperandInfoVector ConstraintOperands;
6838
6839 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6840 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
6841 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
6842
6843 bool hasMemory = false;
6844
6845 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6846 ExtraFlags ExtraInfo(CS);
6847
6848 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6849 unsigned ResNo = 0; // ResNo - The result number of the next output.
6850 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
6851 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
6852 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
6853
6854 MVT OpVT = MVT::Other;
6855
6856 // Compute the value type for each operand.
6857 if (OpInfo.Type == InlineAsm::isInput ||
6858 (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
6859 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
6860
6861 // Process the call argument. BasicBlocks are labels, currently appearing
6862 // only in asm's.
6863 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
6864 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
6865 } else {
6866 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
6867 }
6868
6869 OpVT =
6870 OpInfo
6871 .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
6872 .getSimpleVT();
6873 }
6874
6875 if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
6876 // The return value of the call is this value. As such, there is no
6877 // corresponding argument.
6878 assert(!CS.getType()->isVoidTy() && "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6878, __PRETTY_FUNCTION__))
;
6879 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
6880 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
6881 STy->getElementType(ResNo));
6882 } else {
6883 assert(ResNo == 0 && "Asm only has one result!")((ResNo == 0 && "Asm only has one result!") ? static_cast
<void> (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6883, __PRETTY_FUNCTION__))
;
6884 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
6885 }
6886 ++ResNo;
6887 }
6888
6889 OpInfo.ConstraintVT = OpVT;
6890
6891 if (!hasMemory)
6892 hasMemory = OpInfo.hasMemory(TLI);
6893
6894 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
6895 // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
6896 auto TargetConstraint = TargetConstraints[i];
6897
6898 // Compute the constraint code and ConstraintType to use.
6899 TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
6900
6901 ExtraInfo.update(TargetConstraint);
6902 }
6903
6904 SDValue Chain, Flag;
6905
6906 // We won't need to flush pending loads if this asm doesn't touch
6907 // memory and is nonvolatile.
6908 if (hasMemory || IA->hasSideEffects())
6909 Chain = getRoot();
6910 else
6911 Chain = DAG.getRoot();
6912
6913 // Second pass over the constraints: compute which constraint option to use
6914 // and assign registers to constraints that want a specific physreg.
6915 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6916 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6917
6918 // If this is an output operand with a matching input operand, look up the
6919 // matching input. If their types mismatch, e.g. one is an integer, the
6920 // other is floating point, or their sizes are different, flag it as an
6921 // error.
6922 if (OpInfo.hasMatchingInput()) {
6923 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6924 patchMatchingInput(OpInfo, Input, DAG);
6925 }
6926
6927 // Compute the constraint code and ConstraintType to use.
6928 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
6929
6930 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6931 OpInfo.Type == InlineAsm::isClobber)
6932 continue;
6933
6934 // If this is a memory input, and if the operand is not indirect, do what we
6935 // need to to provide an address for the memory input.
6936 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
6937 !OpInfo.isIndirect) {
6938 assert((OpInfo.isMultipleAlternative ||(((OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::
isInput)) && "Can only indirectify direct input operands!"
) ? static_cast<void> (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6940, __PRETTY_FUNCTION__))
6939 (OpInfo.Type == InlineAsm::isInput)) &&(((OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::
isInput)) && "Can only indirectify direct input operands!"
) ? static_cast<void> (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6940, __PRETTY_FUNCTION__))
6940 "Can only indirectify direct input operands!")(((OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::
isInput)) && "Can only indirectify direct input operands!"
) ? static_cast<void> (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6940, __PRETTY_FUNCTION__))
;
6941
6942 // Memory operands really want the address of the value.
6943 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
6944
6945 // There is no longer a Value* corresponding to this operand.
6946 OpInfo.CallOperandVal = nullptr;
6947
6948 // It is now an indirect operand.
6949 OpInfo.isIndirect = true;
6950 }
6951
6952 // If this constraint is for a specific register, allocate it before
6953 // anything else.
6954 if (OpInfo.ConstraintType == TargetLowering::C_Register)
6955 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6956 }
6957
6958 // Third pass - Loop over all of the operands, assigning virtual or physregs
6959 // to register class operands.
6960 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6961 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6962
6963 // C_Register operands have already been allocated, Other/Memory don't need
6964 // to be.
6965 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
6966 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
6967 }
6968
6969 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
6970 std::vector<SDValue> AsmNodeOperands;
6971 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
6972 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
6973 IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
6974
6975 // If we have a !srcloc metadata node associated with it, we want to attach
6976 // this to the ultimately generated inline asm machineinstr. To do this, we
6977 // pass in the third operand as this (potentially null) inline asm MDNode.
6978 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
6979 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
6980
6981 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
6982 // bits as operand 3.
6983 AsmNodeOperands.push_back(DAG.getTargetConstant(
6984 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
6985
6986 // Loop over all of the inputs, copying the operand values into the
6987 // appropriate registers and processing the output regs.
6988 RegsForValue RetValRegs;
6989
6990 // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
6991 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit;
6992
6993 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
6994 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
6995
6996 switch (OpInfo.Type) {
6997 case InlineAsm::isOutput: {
6998 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
6999 OpInfo.ConstraintType != TargetLowering::C_Register) {
7000 // Memory output, or 'other' output (e.g. 'X' constraint).
7001 assert(OpInfo.isIndirect && "Memory output must be indirect operand")((OpInfo.isIndirect && "Memory output must be indirect operand"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.isIndirect && \"Memory output must be indirect operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7001, __PRETTY_FUNCTION__))
;
7002
7003 unsigned ConstraintID =
7004 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7005 assert(ConstraintID != InlineAsm::Constraint_Unknown &&((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7006, __PRETTY_FUNCTION__))
7006 "Failed to convert memory constraint code to constraint id.")((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7006, __PRETTY_FUNCTION__))
;
7007
7008 // Add information to the INLINEASM node to know about this output.
7009 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7010 OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7011 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7012 MVT::i32));
7013 AsmNodeOperands.push_back(OpInfo.CallOperand);
7014 break;
7015 }
7016
7017 // Otherwise, this is a register or register class output.
7018
7019 // Copy the output from the appropriate register. Find a register that
7020 // we can use.
7021 if (OpInfo.AssignedRegs.Regs.empty()) {
7022 emitInlineAsmError(
7023 CS, "couldn't allocate output register for constraint '" +
7024 Twine(OpInfo.ConstraintCode) + "'");
7025 return;
7026 }
7027
7028 // If this is an indirect operand, store through the pointer after the
7029 // asm.
7030 if (OpInfo.isIndirect) {
7031 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7032 OpInfo.CallOperandVal));
7033 } else {
7034 // This is the result value of the call.
7035 assert(!CS.getType()->isVoidTy() && "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7035, __PRETTY_FUNCTION__))
;
7036 // Concatenate this output onto the outputs list.
7037 RetValRegs.append(OpInfo.AssignedRegs);
7038 }
7039
7040 // Add information to the INLINEASM node to know that this register is
7041 // set.
7042 OpInfo.AssignedRegs
7043 .AddInlineAsmOperands(OpInfo.isEarlyClobber
7044 ? InlineAsm::Kind_RegDefEarlyClobber
7045 : InlineAsm::Kind_RegDef,
7046 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7047 break;
7048 }
7049 case InlineAsm::isInput: {
7050 SDValue InOperandVal = OpInfo.CallOperand;
7051
7052 if (OpInfo.isMatchingInputConstraint()) {
7053 // If this is required to match an output register we have already set,
7054 // just use its register.
7055 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7056 AsmNodeOperands);
7057 unsigned OpFlag =
7058 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7059 if (InlineAsm::isRegDefKind(OpFlag) ||
7060 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7061 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7062 if (OpInfo.isIndirect) {
7063 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7064 emitInlineAsmError(CS, "inline asm not supported yet:"
7065 " don't know how to handle tied "
7066 "indirect register inputs");
7067 return;
7068 }
7069
7070 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7071 SmallVector<unsigned, 4> Regs;
7072
7073 if (!createVirtualRegs(Regs,
7074 InlineAsm::getNumOperandRegisters(OpFlag),
7075 RegVT, DAG)) {
7076 emitInlineAsmError(CS, "inline asm error: This value type register "
7077 "class is not natively supported!");
7078 return;
7079 }
7080
7081 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7082
7083 SDLoc dl = getCurSDLoc();
7084 // Use the produced MatchedRegs object to
7085 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7086 Chain, &Flag, CS.getInstruction());
7087 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7088 true, OpInfo.getMatchedOperand(), dl,
7089 DAG, AsmNodeOperands);
7090 break;
7091 }
7092
7093 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!")((InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::isMemKind(OpFlag) && \"Unknown matching constraint!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7093, __PRETTY_FUNCTION__))
;
7094 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&((InlineAsm::getNumOperandRegisters(OpFlag) == 1 && "Unexpected number of operands"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7095, __PRETTY_FUNCTION__))
7095 "Unexpected number of operands")((InlineAsm::getNumOperandRegisters(OpFlag) == 1 && "Unexpected number of operands"
) ? static_cast<void> (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7095, __PRETTY_FUNCTION__))
;
7096 // Add information to the INLINEASM node to know about this input.
7097 // See InlineAsm.h isUseOperandTiedToDef.
7098 OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7099 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7100 OpInfo.getMatchedOperand());
7101 AsmNodeOperands.push_back(DAG.getTargetConstant(
7102 OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7103 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7104 break;
7105 }
7106
7107 // Treat indirect 'X' constraint as memory.
7108 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7109 OpInfo.isIndirect)
7110 OpInfo.ConstraintType = TargetLowering::C_Memory;
7111
7112 if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7113 std::vector<SDValue> Ops;
7114 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7115 Ops, DAG);
7116 if (Ops.empty()) {
7117 emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7118 Twine(OpInfo.ConstraintCode) + "'");
7119 return;
7120 }
7121
7122 // Add information to the INLINEASM node to know about this input.
7123 unsigned ResOpType =
7124 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7125 AsmNodeOperands.push_back(DAG.getTargetConstant(
7126 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7127 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7128 break;
7129 }
7130
7131 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7132 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!")((OpInfo.isIndirect && "Operand must be indirect to be a mem!"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.isIndirect && \"Operand must be indirect to be a mem!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7132, __PRETTY_FUNCTION__))
;
7133 assert(InOperandVal.getValueType() ==((InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout
()) && "Memory operands expect pointer values") ? static_cast
<void> (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7135, __PRETTY_FUNCTION__))
7134 TLI.getPointerTy(DAG.getDataLayout()) &&((InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout
()) && "Memory operands expect pointer values") ? static_cast
<void> (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7135, __PRETTY_FUNCTION__))
7135 "Memory operands expect pointer values")((InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout
()) && "Memory operands expect pointer values") ? static_cast
<void> (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7135, __PRETTY_FUNCTION__))
;
7136
7137 unsigned ConstraintID =
7138 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7139 assert(ConstraintID != InlineAsm::Constraint_Unknown &&((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7140, __PRETTY_FUNCTION__))
7140 "Failed to convert memory constraint code to constraint id.")((ConstraintID != InlineAsm::Constraint_Unknown && "Failed to convert memory constraint code to constraint id."
) ? static_cast<void> (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7140, __PRETTY_FUNCTION__))
;
7141
7142 // Add information to the INLINEASM node to know about this input.
7143 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7144 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7145 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7146 getCurSDLoc(),
7147 MVT::i32));
7148 AsmNodeOperands.push_back(InOperandVal);
7149 break;
7150 }
7151
7152 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||(((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
OpInfo.ConstraintType == TargetLowering::C_Register) &&
"Unknown constraint type!") ? static_cast<void> (0) : __assert_fail
("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7154, __PRETTY_FUNCTION__))
7153 OpInfo.ConstraintType == TargetLowering::C_Register) &&(((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
OpInfo.ConstraintType == TargetLowering::C_Register) &&
"Unknown constraint type!") ? static_cast<void> (0) : __assert_fail
("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7154, __PRETTY_FUNCTION__))
7154 "Unknown constraint type!")(((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||
OpInfo.ConstraintType == TargetLowering::C_Register) &&
"Unknown constraint type!") ? static_cast<void> (0) : __assert_fail
("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7154, __PRETTY_FUNCTION__))
;
7155
7156 // TODO: Support this.
7157 if (OpInfo.isIndirect) {
7158 emitInlineAsmError(
7159 CS, "Don't know how to handle indirect register inputs yet "
7160 "for constraint '" +
7161 Twine(OpInfo.ConstraintCode) + "'");
7162 return;
7163 }
7164
7165 // Copy the input into the appropriate registers.
7166 if (OpInfo.AssignedRegs.Regs.empty()) {
7167 emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7168 Twine(OpInfo.ConstraintCode) + "'");
7169 return;
7170 }
7171
7172 SDLoc dl = getCurSDLoc();
7173
7174 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7175 Chain, &Flag, CS.getInstruction());
7176
7177 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7178 dl, DAG, AsmNodeOperands);
7179 break;
7180 }
7181 case InlineAsm::isClobber: {
7182 // Add the clobbered value to the operand list, so that the register
7183 // allocator is aware that the physreg got clobbered.
7184 if (!OpInfo.AssignedRegs.Regs.empty())
7185 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7186 false, 0, getCurSDLoc(), DAG,
7187 AsmNodeOperands);
7188 break;
7189 }
7190 }
7191 }
7192
7193 // Finish up input operands. Set the input chain and add the flag last.
7194 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7195 if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7196
7197 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7198 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7199 Flag = Chain.getValue(1);
7200
7201 // If this asm returns a register value, copy the result from that register
7202 // and set it as the value of the call.
7203 if (!RetValRegs.Regs.empty()) {
7204 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7205 Chain, &Flag, CS.getInstruction());
7206
7207 // FIXME: Why don't we do this for inline asms with MRVs?
7208 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
7209 EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7210
7211 // If any of the results of the inline asm is a vector, it may have the
7212 // wrong width/num elts. This can happen for register classes that can
7213 // contain multiple different value types. The preg or vreg allocated may
7214 // not have the same VT as was expected. Convert it to the right type
7215 // with bit_convert.
7216 if (ResultType != Val.getValueType() && Val.getValueType().isVector()) {
7217 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(),
7218 ResultType, Val);
7219
7220 } else if (ResultType != Val.getValueType() &&
7221 ResultType.isInteger() && Val.getValueType().isInteger()) {
7222 // If a result value was tied to an input value, the computed result may
7223 // have a wider width than the expected result. Extract the relevant
7224 // portion.
7225 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
7226 }
7227
7228 assert(ResultType == Val.getValueType() && "Asm result value mismatch!")((ResultType == Val.getValueType() && "Asm result value mismatch!"
) ? static_cast<void> (0) : __assert_fail ("ResultType == Val.getValueType() && \"Asm result value mismatch!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7228, __PRETTY_FUNCTION__))
;
7229 }
7230
7231 setValue(CS.getInstruction(), Val);
7232 // Don't need to use this as a chain in this case.
7233 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7234 return;
7235 }
7236
7237 std::vector<std::pair<SDValue, const Value *> > StoresToEmit;
7238
7239 // Process indirect outputs, first output all of the flagged copies out of
7240 // physregs.
7241 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7242 RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7243 const Value *Ptr = IndirectStoresToEmit[i].second;
7244 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7245 Chain, &Flag, IA);
7246 StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7247 }
7248
7249 // Emit the non-flagged stores from the physregs.
7250 SmallVector<SDValue, 8> OutChains;
7251 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7252 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7253 getValue(StoresToEmit[i].second),
7254 MachinePointerInfo(StoresToEmit[i].second));
7255 OutChains.push_back(Val);
7256 }
7257
7258 if (!OutChains.empty())
7259 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7260
7261 DAG.setRoot(Chain);
7262}
7263
7264void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7265 const Twine &Message) {
7266 LLVMContext &Ctx = *DAG.getContext();
7267 Ctx.emitError(CS.getInstruction(), Message);
7268
7269 // Make sure we leave the DAG in a valid state
7270 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7271 auto VT = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7272 setValue(CS.getInstruction(), DAG.getUNDEF(VT));
7273}
7274
7275void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7276 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
7277 MVT::Other, getRoot(),
7278 getValue(I.getArgOperand(0)),
7279 DAG.getSrcValue(I.getArgOperand(0))));
7280}
7281
7282void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7283 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7284 const DataLayout &DL = DAG.getDataLayout();
7285 SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7286 getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7287 DAG.getSrcValue(I.getOperand(0)),
7288 DL.getABITypeAlignment(I.getType()));
7289 setValue(&I, V);
7290 DAG.setRoot(V.getValue(1));
7291}
7292
7293void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
7294 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
7295 MVT::Other, getRoot(),
7296 getValue(I.getArgOperand(0)),
7297 DAG.getSrcValue(I.getArgOperand(0))));
7298}
7299
7300void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
7301 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
7302 MVT::Other, getRoot(),
7303 getValue(I.getArgOperand(0)),
7304 getValue(I.getArgOperand(1)),
7305 DAG.getSrcValue(I.getArgOperand(0)),
7306 DAG.getSrcValue(I.getArgOperand(1))));
7307}
7308
7309SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
7310 const Instruction &I,
7311 SDValue Op) {
7312 const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
7313 if (!Range)
7314 return Op;
7315
7316 ConstantRange CR = getConstantRangeFromMetadata(*Range);
7317 if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
7318 return Op;
7319
7320 APInt Lo = CR.getUnsignedMin();
7321 if (!Lo.isMinValue())
7322 return Op;
7323
7324 APInt Hi = CR.getUnsignedMax();
7325 unsigned Bits = Hi.getActiveBits();
7326
7327 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
7328
7329 SDLoc SL = getCurSDLoc();
7330
7331 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
7332 DAG.getValueType(SmallVT));
7333 unsigned NumVals = Op.getNode()->getNumValues();
7334 if (NumVals == 1)
7335 return ZExt;
7336
7337 SmallVector<SDValue, 4> Ops;
7338
7339 Ops.push_back(ZExt);
7340 for (unsigned I = 1; I != NumVals; ++I)
7341 Ops.push_back(Op.getValue(I));
7342
7343 return DAG.getMergeValues(Ops, SL);
7344}
7345
7346/// \brief Populate a CallLowerinInfo (into \p CLI) based on the properties of
7347/// the call being lowered.
7348///
7349/// This is a helper for lowering intrinsics that follow a target calling
7350/// convention or require stack pointer adjustment. Only a subset of the
7351/// intrinsic's operands need to participate in the calling convention.
7352void SelectionDAGBuilder::populateCallLoweringInfo(
7353 TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
7354 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
7355 bool IsPatchPoint) {
7356 TargetLowering::ArgListTy Args;
7357 Args.reserve(NumArgs);
7358
7359 // Populate the argument list.
7360 // Attributes for args start at offset 1, after the return attribute.
7361 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
7362 ArgI != ArgE; ++ArgI) {
7363 const Value *V = CS->getOperand(ArgI);
7364
7365 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.")((!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isEmptyTy() && \"Empty type passed to intrinsic.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7365, __PRETTY_FUNCTION__))
;
7366
7367 TargetLowering::ArgListEntry Entry;
7368 Entry.Node = getValue(V);
7369 Entry.Ty = V->getType();
7370 Entry.setAttributes(&CS, ArgIdx);
7371 Args.push_back(Entry);
7372 }
7373
7374 CLI.setDebugLoc(getCurSDLoc())
7375 .setChain(getRoot())
7376 .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
7377 .setDiscardResult(CS->use_empty())
7378 .setIsPatchPoint(IsPatchPoint);
7379}
7380
7381/// \brief Add a stack map intrinsic call's live variable operands to a stackmap
7382/// or patchpoint target node's operand list.
7383///
7384/// Constants are converted to TargetConstants purely as an optimization to
7385/// avoid constant materialization and register allocation.
7386///
7387/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
7388/// generate addess computation nodes, and so ExpandISelPseudo can convert the
7389/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
7390/// address materialization and register allocation, but may also be required
7391/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
7392/// alloca in the entry block, then the runtime may assume that the alloca's
7393/// StackMap location can be read immediately after compilation and that the
7394/// location is valid at any point during execution (this is similar to the
7395/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
7396/// only available in a register, then the runtime would need to trap when
7397/// execution reaches the StackMap in order to read the alloca's location.
7398static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
7399 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
7400 SelectionDAGBuilder &Builder) {
7401 for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
7402 SDValue OpVal = Builder.getValue(CS.getArgument(i));
7403 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
7404 Ops.push_back(
7405 Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
7406 Ops.push_back(
7407 Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
7408 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
7409 const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
7410 Ops.push_back(Builder.DAG.getTargetFrameIndex(
7411 FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
7412 } else
7413 Ops.push_back(OpVal);
7414 }
7415}
7416
7417/// \brief Lower llvm.experimental.stackmap directly to its target opcode.
7418void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
7419 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
7420 // [live variables...])
7421
7422 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.")((CI.getType()->isVoidTy() && "Stackmap cannot return a value."
) ? static_cast<void> (0) : __assert_fail ("CI.getType()->isVoidTy() && \"Stackmap cannot return a value.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7422, __PRETTY_FUNCTION__))
;
7423
7424 SDValue Chain, InFlag, Callee, NullPtr;
7425 SmallVector<SDValue, 32> Ops;
7426
7427 SDLoc DL = getCurSDLoc();
7428 Callee = getValue(CI.getCalledValue());
7429 NullPtr = DAG.getIntPtrConstant(0, DL, true);
7430
7431 // The stackmap intrinsic only records the live variables (the arguemnts
7432 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
7433 // intrinsic, this won't be lowered to a function call. This means we don't
7434 // have to worry about calling conventions and target specific lowering code.
7435 // Instead we perform the call lowering right here.
7436 //
7437 // chain, flag = CALLSEQ_START(chain, 0, 0)
7438 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
7439 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
7440 //
7441 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
7442 InFlag = Chain.getValue(1);
7443
7444 // Add the <id> and <numBytes> constants.
7445 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
7446 Ops.push_back(DAG.getTargetConstant(
7447 cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
7448 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
7449 Ops.push_back(DAG.getTargetConstant(
7450 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
7451 MVT::i32));
7452
7453 // Push live variables for the stack map.
7454 addStackMapLiveVars(&CI, 2, DL, Ops, *this);
7455
7456 // We are not pushing any register mask info here on the operands list,
7457 // because the stackmap doesn't clobber anything.
7458
7459 // Push the chain and the glue flag.
7460 Ops.push_back(Chain);
7461 Ops.push_back(InFlag);
7462
7463 // Create the STACKMAP node.
7464 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7465 SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
7466 Chain = SDValue(SM, 0);
7467 InFlag = Chain.getValue(1);
7468
7469 Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
7470
7471 // Stackmaps don't generate values, so nothing goes into the NodeMap.
7472
7473 // Set the root to the target-lowered call chain.
7474 DAG.setRoot(Chain);
7475
7476 // Inform the Frame Information that we have a stackmap in this function.
7477 FuncInfo.MF->getFrameInfo().setHasStackMap();
7478}
7479
7480/// \brief Lower llvm.experimental.patchpoint directly to its target opcode.
7481void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
7482 const BasicBlock *EHPadBB) {
7483 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
7484 // i32 <numBytes>,
7485 // i8* <target>,
7486 // i32 <numArgs>,
7487 // [Args...],
7488 // [live variables...])
7489
7490 CallingConv::ID CC = CS.getCallingConv();
7491 bool IsAnyRegCC = CC == CallingConv::AnyReg;
7492 bool HasDef = !CS->getType()->isVoidTy();
7493 SDLoc dl = getCurSDLoc();
7494 SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
7495
7496 // Handle immediate and symbolic callees.
7497 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
7498 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
7499 /*isTarget=*/true);
7500 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
7501 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
7502 SDLoc(SymbolicCallee),
7503 SymbolicCallee->getValueType(0));
7504
7505 // Get the real number of arguments participating in the call <numArgs>
7506 SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
7507 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
7508
7509 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
7510 // Intrinsics include all meta-operands up to but not including CC.
7511 unsigned NumMetaOpers = PatchPointOpers::CCPos;
7512 assert(CS.arg_size() >= NumMetaOpers + NumArgs &&((CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? static_cast<void> (0) : __assert_fail ("CS.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7513, __PRETTY_FUNCTION__))
7513 "Not enough arguments provided to the patchpoint intrinsic")((CS.arg_size() >= NumMetaOpers + NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? static_cast<void> (0) : __assert_fail ("CS.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7513, __PRETTY_FUNCTION__))
;
7514
7515 // For AnyRegCC the arguments are lowered later on manually.
7516 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
7517 Type *ReturnTy =
7518 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
7519
7520 TargetLowering::CallLoweringInfo CLI(DAG);
7521 populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
7522 true);
7523 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
7524
7525 SDNode *CallEnd = Result.second.getNode();
7526 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
7527 CallEnd = CallEnd->getOperand(0).getNode();
7528
7529 /// Get a call instruction from the call sequence chain.
7530 /// Tail calls are not allowed.
7531 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&((CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node."
) ? static_cast<void> (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7532, __PRETTY_FUNCTION__))
7532 "Expected a callseq node.")((CallEnd->getOpcode() == ISD::CALLSEQ_END && "Expected a callseq node."
) ? static_cast<void> (0) : __assert_fail ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7532, __PRETTY_FUNCTION__))
;
7533 SDNode *Call = CallEnd->getOperand(0).getNode();
7534 bool HasGlue = Call->getGluedNode();
7535
7536 // Replace the target specific call node with the patchable intrinsic.
7537 SmallVector<SDValue, 8> Ops;
7538
7539 // Add the <id> and <numBytes> constants.
7540 SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
7541 Ops.push_back(DAG.getTargetConstant(
7542 cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
7543 SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
7544 Ops.push_back(DAG.getTargetConstant(
7545 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
7546 MVT::i32));
7547
7548 // Add the callee.
7549 Ops.push_back(Callee);
7550
7551 // Adjust <numArgs> to account for any arguments that have been passed on the
7552 // stack instead.
7553 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
7554 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
7555 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
7556 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
7557
7558 // Add the calling convention
7559 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
7560
7561 // Add the arguments we omitted previously. The register allocator should
7562 // place these in any free register.
7563 if (IsAnyRegCC)
7564 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
7565 Ops.push_back(getValue(CS.getArgument(i)));
7566
7567 // Push the arguments from the call instruction up to the register mask.
7568 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
7569 Ops.append(Call->op_begin() + 2, e);
7570
7571 // Push live variables for the stack map.
7572 addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
7573
7574 // Push the register mask info.
7575 if (HasGlue)
7576 Ops.push_back(*(Call->op_end()-2));
7577 else
7578 Ops.push_back(*(Call->op_end()-1));
7579
7580 // Push the chain (this is originally the first operand of the call, but
7581 // becomes now the last or second to last operand).
7582 Ops.push_back(*(Call->op_begin()));
7583
7584 // Push the glue flag (last operand).
7585 if (HasGlue)
7586 Ops.push_back(*(Call->op_end()-1));
7587
7588 SDVTList NodeTys;
7589 if (IsAnyRegCC && HasDef) {
7590 // Create the return types based on the intrinsic definition
7591 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7592 SmallVector<EVT, 3> ValueVTs;
7593 ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7594 assert(ValueVTs.size() == 1 && "Expected only one return value type.")((ValueVTs.size() == 1 && "Expected only one return value type."
) ? static_cast<void> (0) : __assert_fail ("ValueVTs.size() == 1 && \"Expected only one return value type.\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7594, __PRETTY_FUNCTION__))
;
7595
7596 // There is always a chain and a glue type at the end
7597 ValueVTs.push_back(MVT::Other);
7598 ValueVTs.push_back(MVT::Glue);
7599 NodeTys = DAG.getVTList(ValueVTs);
7600 } else
7601 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7602
7603 // Replace the target specific call node with a PATCHPOINT node.
7604 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
7605 dl, NodeTys, Ops);
7606
7607 // Update the NodeMap.
7608 if (HasDef) {
7609 if (IsAnyRegCC)
7610 setValue(CS.getInstruction(), SDValue(MN, 0));
7611 else
7612 setValue(CS.getInstruction(), Result.first);
7613 }
7614
7615 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
7616 // call sequence. Furthermore the location of the chain and glue can change
7617 // when the AnyReg calling convention is used and the intrinsic returns a
7618 // value.
7619 if (IsAnyRegCC && HasDef) {
7620 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
7621 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
7622 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
7623 } else
7624 DAG.ReplaceAllUsesWith(Call, MN);
7625 DAG.DeleteNode(Call);
7626
7627 // Inform the Frame Information that we have a patchpoint in this function.
7628 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
7629}
7630
7631void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
7632 unsigned Intrinsic) {
7633 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7634 SDValue Op1 = getValue(I.getArgOperand(0));
7635 SDValue Op2;
7636 if (I.getNumArgOperands() > 1)
7637 Op2 = getValue(I.getArgOperand(1));
7638 SDLoc dl = getCurSDLoc();
7639 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7640 SDValue Res;
7641 FastMathFlags FMF;
7642 if (isa<FPMathOperator>(I))
7643 FMF = I.getFastMathFlags();
7644 SDNodeFlags SDFlags;
7645 SDFlags.setNoNaNs(FMF.noNaNs());
7646
7647 switch (Intrinsic) {
7648 case Intrinsic::experimental_vector_reduce_fadd:
7649 if (FMF.unsafeAlgebra())
7650 Res = DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2);
7651 else
7652 Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2);
7653 break;
7654 case Intrinsic::experimental_vector_reduce_fmul:
7655 if (FMF.unsafeAlgebra())
7656 Res = DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2);
7657 else
7658 Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2);
7659 break;
7660 case Intrinsic::experimental_vector_reduce_add:
7661 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
7662 break;
7663 case Intrinsic::experimental_vector_reduce_mul:
7664 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
7665 break;
7666 case Intrinsic::experimental_vector_reduce_and:
7667 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
7668 break;
7669 case Intrinsic::experimental_vector_reduce_or:
7670 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
7671 break;
7672 case Intrinsic::experimental_vector_reduce_xor:
7673 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
7674 break;
7675 case Intrinsic::experimental_vector_reduce_smax:
7676 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
7677 break;
7678 case Intrinsic::experimental_vector_reduce_smin:
7679 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
7680 break;
7681 case Intrinsic::experimental_vector_reduce_umax:
7682 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
7683 break;
7684 case Intrinsic::experimental_vector_reduce_umin:
7685 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
7686 break;
7687 case Intrinsic::experimental_vector_reduce_fmax: {
7688 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
7689 break;
7690 }
7691 case Intrinsic::experimental_vector_reduce_fmin: {
7692 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
7693 break;
7694 }
7695 default:
7696 llvm_unreachable("Unhandled vector reduce intrinsic")::llvm::llvm_unreachable_internal("Unhandled vector reduce intrinsic"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7696)
;
7697 }
7698 setValue(&I, Res);
7699}
7700
7701/// Returns an AttributeList representing the attributes applied to the return
7702/// value of the given call.
7703static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
7704 SmallVector<Attribute::AttrKind, 2> Attrs;
7705 if (CLI.RetSExt)
7706 Attrs.push_back(Attribute::SExt);
7707 if (CLI.RetZExt)
7708 Attrs.push_back(Attribute::ZExt);
7709 if (CLI.IsInReg)
7710 Attrs.push_back(Attribute::InReg);
7711
7712 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
7713 Attrs);
7714}
7715
7716/// TargetLowering::LowerCallTo - This is the default LowerCallTo
7717/// implementation, which just calls LowerCall.
7718/// FIXME: When all targets are
7719/// migrated to using LowerCall, this hook should be integrated into SDISel.
7720std::pair<SDValue, SDValue>
7721TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
7722 // Handle the incoming return values from the call.
7723 CLI.Ins.clear();
7724 Type *OrigRetTy = CLI.RetTy;
7725 SmallVector<EVT, 4> RetTys;
7726 SmallVector<uint64_t, 4> Offsets;
7727 auto &DL = CLI.DAG.getDataLayout();
7728 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
7729
7730 SmallVector<ISD::OutputArg, 4> Outs;
7731 GetReturnInfo(CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
7732
7733 bool CanLowerReturn =
7734 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
7735 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
7736
7737 SDValue DemoteStackSlot;
7738 int DemoteStackIdx = -100;
7739 if (!CanLowerReturn) {
7740 // FIXME: equivalent assert?
7741 // assert(!CS.hasInAllocaArgument() &&
7742 // "sret demotion is incompatible with inalloca");
7743 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
7744 unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
7745 MachineFunction &MF = CLI.DAG.getMachineFunction();
7746 DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7747 Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
7748
7749 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
7750 ArgListEntry Entry;
7751 Entry.Node = DemoteStackSlot;
7752 Entry.Ty = StackSlotPtrType;
7753 Entry.IsSExt = false;
7754 Entry.IsZExt = false;
7755 Entry.IsInReg = false;
7756 Entry.IsSRet = true;
7757 Entry.IsNest = false;
7758 Entry.IsByVal = false;