LLVM 17.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
56#include "llvm/IR/Argument.h"
57#include "llvm/IR/Attributes.h"
58#include "llvm/IR/BasicBlock.h"
59#include "llvm/IR/CFG.h"
60#include "llvm/IR/CallingConv.h"
61#include "llvm/IR/Constant.h"
63#include "llvm/IR/Constants.h"
64#include "llvm/IR/DataLayout.h"
65#include "llvm/IR/DebugInfo.h"
70#include "llvm/IR/Function.h"
72#include "llvm/IR/InlineAsm.h"
73#include "llvm/IR/InstrTypes.h"
76#include "llvm/IR/Intrinsics.h"
77#include "llvm/IR/IntrinsicsAArch64.h"
78#include "llvm/IR/IntrinsicsWebAssembly.h"
79#include "llvm/IR/LLVMContext.h"
80#include "llvm/IR/Metadata.h"
81#include "llvm/IR/Module.h"
82#include "llvm/IR/Operator.h"
84#include "llvm/IR/Statepoint.h"
85#include "llvm/IR/Type.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/MC/MCContext.h"
93#include "llvm/Support/Debug.h"
101#include <cstddef>
102#include <iterator>
103#include <limits>
104#include <optional>
105#include <tuple>
106
107using namespace llvm;
108using namespace PatternMatch;
109using namespace SwitchCG;
110
111#define DEBUG_TYPE "isel"
112
113/// LimitFloatPrecision - Generate low-precision inline sequences for
114/// some float libcalls (6, 8 or 12 bits).
115static unsigned LimitFloatPrecision;
116
117static cl::opt<bool>
118 InsertAssertAlign("insert-assert-align", cl::init(true),
119 cl::desc("Insert the experimental `assertalign` node."),
121
123 LimitFPPrecision("limit-float-precision",
124 cl::desc("Generate low-precision inline sequences "
125 "for some float libcalls"),
127 cl::init(0));
128
130 "switch-peel-threshold", cl::Hidden, cl::init(66),
131 cl::desc("Set the case probability threshold for peeling the case from a "
132 "switch statement. A value greater than 100 will void this "
133 "optimization"));
134
135// Limit the width of DAG chains. This is important in general to prevent
136// DAG-based analysis from blowing up. For example, alias analysis and
137// load clustering may not complete in reasonable time. It is difficult to
138// recognize and avoid this situation within each individual analysis, and
139// future analyses are likely to have the same behavior. Limiting DAG width is
140// the safe approach and will be especially important with global DAGs.
141//
142// MaxParallelChains default is arbitrarily high to avoid affecting
143// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
144// sequence over this should have been converted to llvm.memcpy by the
145// frontend. It is easy to induce this behavior with .ll code such as:
146// %buffer = alloca [4096 x i8]
147// %data = load [4096 x i8]* %argPtr
148// store [4096 x i8] %data, [4096 x i8]* %buffer
149static const unsigned MaxParallelChains = 64;
150
152 const SDValue *Parts, unsigned NumParts,
153 MVT PartVT, EVT ValueVT, const Value *V,
154 std::optional<CallingConv::ID> CC);
155
156/// getCopyFromParts - Create a value that contains the specified legal parts
157/// combined into the value they represent. If the parts combine to a type
158/// larger than ValueVT then AssertOp can be used to specify whether the extra
159/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
160/// (ISD::AssertSext).
161static SDValue
162getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
163 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
164 std::optional<CallingConv::ID> CC = std::nullopt,
165 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
166 // Let the target assemble the parts if it wants to
167 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
168 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
169 PartVT, ValueVT, CC))
170 return Val;
171
172 if (ValueVT.isVector())
173 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
174 CC);
175
176 assert(NumParts > 0 && "No parts to assemble!");
177 SDValue Val = Parts[0];
178
179 if (NumParts > 1) {
180 // Assemble the value from multiple parts.
181 if (ValueVT.isInteger()) {
182 unsigned PartBits = PartVT.getSizeInBits();
183 unsigned ValueBits = ValueVT.getSizeInBits();
184
185 // Assemble the power of 2 part.
186 unsigned RoundParts = llvm::bit_floor(NumParts);
187 unsigned RoundBits = PartBits * RoundParts;
188 EVT RoundVT = RoundBits == ValueBits ?
189 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
190 SDValue Lo, Hi;
191
192 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
193
194 if (RoundParts > 2) {
195 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
196 PartVT, HalfVT, V);
197 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
198 RoundParts / 2, PartVT, HalfVT, V);
199 } else {
200 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
201 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
202 }
203
204 if (DAG.getDataLayout().isBigEndian())
205 std::swap(Lo, Hi);
206
207 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
208
209 if (RoundParts < NumParts) {
210 // Assemble the trailing non-power-of-2 part.
211 unsigned OddParts = NumParts - RoundParts;
212 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
213 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
214 OddVT, V, CC);
215
216 // Combine the round and odd parts.
217 Lo = Val;
218 if (DAG.getDataLayout().isBigEndian())
219 std::swap(Lo, Hi);
220 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
221 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
222 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
223 DAG.getConstant(Lo.getValueSizeInBits(), DL,
225 TotalVT, DAG.getDataLayout())));
226 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
227 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
228 }
229 } else if (PartVT.isFloatingPoint()) {
230 // FP split into multiple FP parts (for ppcf128)
231 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
232 "Unexpected split");
233 SDValue Lo, Hi;
234 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
235 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
236 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
237 std::swap(Lo, Hi);
238 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
239 } else {
240 // FP split into integer parts (soft fp)
241 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
242 !PartVT.isVector() && "Unexpected split");
243 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
244 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
245 }
246 }
247
248 // There is now one part, held in Val. Correct it to match ValueVT.
249 // PartEVT is the type of the register class that holds the value.
250 // ValueVT is the type of the inline asm operation.
251 EVT PartEVT = Val.getValueType();
252
253 if (PartEVT == ValueVT)
254 return Val;
255
256 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
257 ValueVT.bitsLT(PartEVT)) {
258 // For an FP value in an integer part, we need to truncate to the right
259 // width first.
260 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
261 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
262 }
263
264 // Handle types that have the same size.
265 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
266 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
267
268 // Handle types with different sizes.
269 if (PartEVT.isInteger() && ValueVT.isInteger()) {
270 if (ValueVT.bitsLT(PartEVT)) {
271 // For a truncate, see if we have any information to
272 // indicate whether the truncated bits will always be
273 // zero or sign-extension.
274 if (AssertOp)
275 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
276 DAG.getValueType(ValueVT));
277 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
278 }
279 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
280 }
281
282 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
283 // FP_ROUND's are always exact here.
284 if (ValueVT.bitsLT(Val.getValueType()))
285 return DAG.getNode(
286 ISD::FP_ROUND, DL, ValueVT, Val,
287 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
288
289 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
290 }
291
292 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
293 // then truncating.
294 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
295 ValueVT.bitsLT(PartEVT)) {
296 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
297 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
298 }
299
300 report_fatal_error("Unknown mismatch in getCopyFromParts!");
301}
302
304 const Twine &ErrMsg) {
305 const Instruction *I = dyn_cast_or_null<Instruction>(V);
306 if (!V)
307 return Ctx.emitError(ErrMsg);
308
309 const char *AsmError = ", possible invalid constraint for vector type";
310 if (const CallInst *CI = dyn_cast<CallInst>(I))
311 if (CI->isInlineAsm())
312 return Ctx.emitError(I, ErrMsg + AsmError);
313
314 return Ctx.emitError(I, ErrMsg);
315}
316
317/// getCopyFromPartsVector - Create a value that contains the specified legal
318/// parts combined into the value they represent. If the parts combine to a
319/// type larger than ValueVT then AssertOp can be used to specify whether the
320/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
321/// ValueVT (ISD::AssertSext).
323 const SDValue *Parts, unsigned NumParts,
324 MVT PartVT, EVT ValueVT, const Value *V,
325 std::optional<CallingConv::ID> CallConv) {
326 assert(ValueVT.isVector() && "Not a vector value");
327 assert(NumParts > 0 && "No parts to assemble!");
328 const bool IsABIRegCopy = CallConv.has_value();
329
330 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
331 SDValue Val = Parts[0];
332
333 // Handle a multi-element vector.
334 if (NumParts > 1) {
335 EVT IntermediateVT;
336 MVT RegisterVT;
337 unsigned NumIntermediates;
338 unsigned NumRegs;
339
340 if (IsABIRegCopy) {
342 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
343 NumIntermediates, RegisterVT);
344 } else {
345 NumRegs =
346 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
347 NumIntermediates, RegisterVT);
348 }
349
350 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
351 NumParts = NumRegs; // Silence a compiler warning.
352 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
353 assert(RegisterVT.getSizeInBits() ==
354 Parts[0].getSimpleValueType().getSizeInBits() &&
355 "Part type sizes don't match!");
356
357 // Assemble the parts into intermediate operands.
358 SmallVector<SDValue, 8> Ops(NumIntermediates);
359 if (NumIntermediates == NumParts) {
360 // If the register was not expanded, truncate or copy the value,
361 // as appropriate.
362 for (unsigned i = 0; i != NumParts; ++i)
363 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
364 PartVT, IntermediateVT, V, CallConv);
365 } else if (NumParts > 0) {
366 // If the intermediate type was expanded, build the intermediate
367 // operands from the parts.
368 assert(NumParts % NumIntermediates == 0 &&
369 "Must expand into a divisible number of parts!");
370 unsigned Factor = NumParts / NumIntermediates;
371 for (unsigned i = 0; i != NumIntermediates; ++i)
372 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
373 PartVT, IntermediateVT, V, CallConv);
374 }
375
376 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
377 // intermediate operands.
378 EVT BuiltVectorTy =
379 IntermediateVT.isVector()
381 *DAG.getContext(), IntermediateVT.getScalarType(),
382 IntermediateVT.getVectorElementCount() * NumParts)
384 IntermediateVT.getScalarType(),
385 NumIntermediates);
386 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
388 DL, BuiltVectorTy, Ops);
389 }
390
391 // There is now one part, held in Val. Correct it to match ValueVT.
392 EVT PartEVT = Val.getValueType();
393
394 if (PartEVT == ValueVT)
395 return Val;
396
397 if (PartEVT.isVector()) {
398 // Vector/Vector bitcast.
399 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
400 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
401
402 // If the parts vector has more elements than the value vector, then we
403 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
404 // Extract the elements we want.
405 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
408 (PartEVT.getVectorElementCount().isScalable() ==
409 ValueVT.getVectorElementCount().isScalable()) &&
410 "Cannot narrow, it would be a lossy transformation");
411 PartEVT =
413 ValueVT.getVectorElementCount());
414 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
415 DAG.getVectorIdxConstant(0, DL));
416 if (PartEVT == ValueVT)
417 return Val;
418 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
419 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
420 }
421
422 // Promoted vector extract
423 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
424 }
425
426 // Trivial bitcast if the types are the same size and the destination
427 // vector type is legal.
428 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
429 TLI.isTypeLegal(ValueVT))
430 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
431
432 if (ValueVT.getVectorNumElements() != 1) {
433 // Certain ABIs require that vectors are passed as integers. For vectors
434 // are the same size, this is an obvious bitcast.
435 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
436 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
437 } else if (ValueVT.bitsLT(PartEVT)) {
438 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
439 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
440 // Drop the extra bits.
441 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
442 return DAG.getBitcast(ValueVT, Val);
443 }
444
446 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
447 return DAG.getUNDEF(ValueVT);
448 }
449
450 // Handle cases such as i8 -> <1 x i1>
451 EVT ValueSVT = ValueVT.getVectorElementType();
452 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
453 unsigned ValueSize = ValueSVT.getSizeInBits();
454 if (ValueSize == PartEVT.getSizeInBits()) {
455 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
456 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
457 // It's possible a scalar floating point type gets softened to integer and
458 // then promoted to a larger integer. If PartEVT is the larger integer
459 // we need to truncate it and then bitcast to the FP type.
460 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
461 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
462 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
463 Val = DAG.getBitcast(ValueSVT, Val);
464 } else {
465 Val = ValueVT.isFloatingPoint()
466 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
467 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
468 }
469 }
470
471 return DAG.getBuildVector(ValueVT, DL, Val);
472}
473
474static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
475 SDValue Val, SDValue *Parts, unsigned NumParts,
476 MVT PartVT, const Value *V,
477 std::optional<CallingConv::ID> CallConv);
478
479/// getCopyToParts - Create a series of nodes that contain the specified value
480/// split into legal parts. If the parts contain more bits than Val, then, for
481/// integers, ExtendKind can be used to specify how to generate the extra bits.
482static void
484 unsigned NumParts, MVT PartVT, const Value *V,
485 std::optional<CallingConv::ID> CallConv = std::nullopt,
486 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
487 // Let the target split the parts if it wants to
488 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
489 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
490 CallConv))
491 return;
492 EVT ValueVT = Val.getValueType();
493
494 // Handle the vector case separately.
495 if (ValueVT.isVector())
496 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
497 CallConv);
498
499 unsigned OrigNumParts = NumParts;
501 "Copying to an illegal type!");
502
503 if (NumParts == 0)
504 return;
505
506 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
507 EVT PartEVT = PartVT;
508 if (PartEVT == ValueVT) {
509 assert(NumParts == 1 && "No-op copy with multiple parts!");
510 Parts[0] = Val;
511 return;
512 }
513
514 unsigned PartBits = PartVT.getSizeInBits();
515 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
516 // If the parts cover more bits than the value has, promote the value.
517 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
518 assert(NumParts == 1 && "Do not know what to promote to!");
519 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
520 } else {
521 if (ValueVT.isFloatingPoint()) {
522 // FP values need to be bitcast, then extended if they are being put
523 // into a larger container.
524 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
525 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
526 }
527 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
528 ValueVT.isInteger() &&
529 "Unknown mismatch!");
530 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
531 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
532 if (PartVT == MVT::x86mmx)
533 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
534 }
535 } else if (PartBits == ValueVT.getSizeInBits()) {
536 // Different types of the same size.
537 assert(NumParts == 1 && PartEVT != ValueVT);
538 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
539 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
540 // If the parts cover less bits than value has, truncate the value.
541 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
542 ValueVT.isInteger() &&
543 "Unknown mismatch!");
544 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
545 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
546 if (PartVT == MVT::x86mmx)
547 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
548 }
549
550 // The value may have changed - recompute ValueVT.
551 ValueVT = Val.getValueType();
552 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
553 "Failed to tile the value with PartVT!");
554
555 if (NumParts == 1) {
556 if (PartEVT != ValueVT) {
558 "scalar-to-vector conversion failed");
559 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
560 }
561
562 Parts[0] = Val;
563 return;
564 }
565
566 // Expand the value into multiple parts.
567 if (NumParts & (NumParts - 1)) {
568 // The number of parts is not a power of 2. Split off and copy the tail.
569 assert(PartVT.isInteger() && ValueVT.isInteger() &&
570 "Do not know what to expand to!");
571 unsigned RoundParts = llvm::bit_floor(NumParts);
572 unsigned RoundBits = RoundParts * PartBits;
573 unsigned OddParts = NumParts - RoundParts;
574 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
575 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
576
577 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
578 CallConv);
579
580 if (DAG.getDataLayout().isBigEndian())
581 // The odd parts were reversed by getCopyToParts - unreverse them.
582 std::reverse(Parts + RoundParts, Parts + NumParts);
583
584 NumParts = RoundParts;
585 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
586 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
587 }
588
589 // The number of parts is a power of 2. Repeatedly bisect the value using
590 // EXTRACT_ELEMENT.
591 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
593 ValueVT.getSizeInBits()),
594 Val);
595
596 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
597 for (unsigned i = 0; i < NumParts; i += StepSize) {
598 unsigned ThisBits = StepSize * PartBits / 2;
599 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
600 SDValue &Part0 = Parts[i];
601 SDValue &Part1 = Parts[i+StepSize/2];
602
603 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
605 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
606 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
607
608 if (ThisBits == PartBits && ThisVT != PartVT) {
609 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
610 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
611 }
612 }
613 }
614
615 if (DAG.getDataLayout().isBigEndian())
616 std::reverse(Parts, Parts + OrigNumParts);
617}
618
620 const SDLoc &DL, EVT PartVT) {
621 if (!PartVT.isVector())
622 return SDValue();
623
624 EVT ValueVT = Val.getValueType();
625 ElementCount PartNumElts = PartVT.getVectorElementCount();
626 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
627
628 // We only support widening vectors with equivalent element types and
629 // fixed/scalable properties. If a target needs to widen a fixed-length type
630 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
631 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
632 PartNumElts.isScalable() != ValueNumElts.isScalable() ||
633 PartVT.getVectorElementType() != ValueVT.getVectorElementType())
634 return SDValue();
635
636 // Widening a scalable vector to another scalable vector is done by inserting
637 // the vector into a larger undef one.
638 if (PartNumElts.isScalable())
639 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
640 Val, DAG.getVectorIdxConstant(0, DL));
641
642 EVT ElementVT = PartVT.getVectorElementType();
643 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
644 // undef elements.
646 DAG.ExtractVectorElements(Val, Ops);
647 SDValue EltUndef = DAG.getUNDEF(ElementVT);
648 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
649
650 // FIXME: Use CONCAT for 2x -> 4x.
651 return DAG.getBuildVector(PartVT, DL, Ops);
652}
653
654/// getCopyToPartsVector - Create a series of nodes that contain the specified
655/// value split into legal parts.
656static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
657 SDValue Val, SDValue *Parts, unsigned NumParts,
658 MVT PartVT, const Value *V,
659 std::optional<CallingConv::ID> CallConv) {
660 EVT ValueVT = Val.getValueType();
661 assert(ValueVT.isVector() && "Not a vector");
662 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
663 const bool IsABIRegCopy = CallConv.has_value();
664
665 if (NumParts == 1) {
666 EVT PartEVT = PartVT;
667 if (PartEVT == ValueVT) {
668 // Nothing to do.
669 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
670 // Bitconvert vector->vector case.
671 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
672 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
673 Val = Widened;
674 } else if (PartVT.isVector() &&
676 ValueVT.getVectorElementType()) &&
677 PartEVT.getVectorElementCount() ==
678 ValueVT.getVectorElementCount()) {
679
680 // Promoted vector extract
681 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
682 } else if (PartEVT.isVector() &&
683 PartEVT.getVectorElementType() !=
684 ValueVT.getVectorElementType() &&
685 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
686 TargetLowering::TypeWidenVector) {
687 // Combination of widening and promotion.
688 EVT WidenVT =
690 PartVT.getVectorElementCount());
691 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
692 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
693 } else {
694 // Don't extract an integer from a float vector. This can happen if the
695 // FP type gets softened to integer and then promoted. The promotion
696 // prevents it from being picked up by the earlier bitcast case.
697 if (ValueVT.getVectorElementCount().isScalar() &&
698 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
699 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
700 DAG.getVectorIdxConstant(0, DL));
701 } else {
702 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
703 assert(PartVT.getFixedSizeInBits() > ValueSize &&
704 "lossy conversion of vector to scalar type");
705 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
706 Val = DAG.getBitcast(IntermediateType, Val);
707 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
708 }
709 }
710
711 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
712 Parts[0] = Val;
713 return;
714 }
715
716 // Handle a multi-element vector.
717 EVT IntermediateVT;
718 MVT RegisterVT;
719 unsigned NumIntermediates;
720 unsigned NumRegs;
721 if (IsABIRegCopy) {
723 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
724 RegisterVT);
725 } else {
726 NumRegs =
727 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
728 NumIntermediates, RegisterVT);
729 }
730
731 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
732 NumParts = NumRegs; // Silence a compiler warning.
733 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
734
735 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
736 "Mixing scalable and fixed vectors when copying in parts");
737
738 std::optional<ElementCount> DestEltCnt;
739
740 if (IntermediateVT.isVector())
741 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
742 else
743 DestEltCnt = ElementCount::getFixed(NumIntermediates);
744
745 EVT BuiltVectorTy = EVT::getVectorVT(
746 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
747
748 if (ValueVT == BuiltVectorTy) {
749 // Nothing to do.
750 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
751 // Bitconvert vector->vector case.
752 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
753 } else {
754 if (BuiltVectorTy.getVectorElementType().bitsGT(
755 ValueVT.getVectorElementType())) {
756 // Integer promotion.
757 ValueVT = EVT::getVectorVT(*DAG.getContext(),
758 BuiltVectorTy.getVectorElementType(),
759 ValueVT.getVectorElementCount());
760 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
761 }
762
763 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
764 Val = Widened;
765 }
766 }
767
768 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
769
770 // Split the vector into intermediate operands.
771 SmallVector<SDValue, 8> Ops(NumIntermediates);
772 for (unsigned i = 0; i != NumIntermediates; ++i) {
773 if (IntermediateVT.isVector()) {
774 // This does something sensible for scalable vectors - see the
775 // definition of EXTRACT_SUBVECTOR for further details.
776 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
777 Ops[i] =
778 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
779 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
780 } else {
781 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
782 DAG.getVectorIdxConstant(i, DL));
783 }
784 }
785
786 // Split the intermediate operands into legal parts.
787 if (NumParts == NumIntermediates) {
788 // If the register was not expanded, promote or copy the value,
789 // as appropriate.
790 for (unsigned i = 0; i != NumParts; ++i)
791 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
792 } else if (NumParts > 0) {
793 // If the intermediate type was expanded, split each the value into
794 // legal parts.
795 assert(NumIntermediates != 0 && "division by zero");
796 assert(NumParts % NumIntermediates == 0 &&
797 "Must expand into a divisible number of parts!");
798 unsigned Factor = NumParts / NumIntermediates;
799 for (unsigned i = 0; i != NumIntermediates; ++i)
800 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
801 CallConv);
802 }
803}
804
806 EVT valuevt, std::optional<CallingConv::ID> CC)
807 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
808 RegCount(1, regs.size()), CallConv(CC) {}
809
811 const DataLayout &DL, unsigned Reg, Type *Ty,
812 std::optional<CallingConv::ID> CC) {
813 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
814
815 CallConv = CC;
816
817 for (EVT ValueVT : ValueVTs) {
818 unsigned NumRegs =
821 : TLI.getNumRegisters(Context, ValueVT);
822 MVT RegisterVT =
825 : TLI.getRegisterType(Context, ValueVT);
826 for (unsigned i = 0; i != NumRegs; ++i)
827 Regs.push_back(Reg + i);
828 RegVTs.push_back(RegisterVT);
829 RegCount.push_back(NumRegs);
830 Reg += NumRegs;
831 }
832}
833
836 const SDLoc &dl, SDValue &Chain,
837 SDValue *Flag, const Value *V) const {
838 // A Value with type {} or [0 x %t] needs no registers.
839 if (ValueVTs.empty())
840 return SDValue();
841
842 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
843
844 // Assemble the legal parts into the final values.
845 SmallVector<SDValue, 4> Values(ValueVTs.size());
847 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
848 // Copy the legal parts from the registers.
849 EVT ValueVT = ValueVTs[Value];
850 unsigned NumRegs = RegCount[Value];
851 MVT RegisterVT = isABIMangled()
853 *DAG.getContext(), *CallConv, RegVTs[Value])
854 : RegVTs[Value];
855
856 Parts.resize(NumRegs);
857 for (unsigned i = 0; i != NumRegs; ++i) {
858 SDValue P;
859 if (!Flag) {
860 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
861 } else {
862 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
863 *Flag = P.getValue(2);
864 }
865
866 Chain = P.getValue(1);
867 Parts[i] = P;
868
869 // If the source register was virtual and if we know something about it,
870 // add an assert node.
871 if (!Register::isVirtualRegister(Regs[Part + i]) ||
872 !RegisterVT.isInteger())
873 continue;
874
876 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
877 if (!LOI)
878 continue;
879
880 unsigned RegSize = RegisterVT.getScalarSizeInBits();
881 unsigned NumSignBits = LOI->NumSignBits;
882 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
883
884 if (NumZeroBits == RegSize) {
885 // The current value is a zero.
886 // Explicitly express that as it would be easier for
887 // optimizations to kick in.
888 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
889 continue;
890 }
891
892 // FIXME: We capture more information than the dag can represent. For
893 // now, just use the tightest assertzext/assertsext possible.
894 bool isSExt;
895 EVT FromVT(MVT::Other);
896 if (NumZeroBits) {
897 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
898 isSExt = false;
899 } else if (NumSignBits > 1) {
900 FromVT =
901 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
902 isSExt = true;
903 } else {
904 continue;
905 }
906 // Add an assertion node.
907 assert(FromVT != MVT::Other);
908 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
909 RegisterVT, P, DAG.getValueType(FromVT));
910 }
911
912 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
913 RegisterVT, ValueVT, V, CallConv);
914 Part += NumRegs;
915 Parts.clear();
916 }
917
918 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
919}
920
922 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
923 const Value *V,
924 ISD::NodeType PreferredExtendType) const {
925 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
926 ISD::NodeType ExtendKind = PreferredExtendType;
927
928 // Get the list of the values's legal parts.
929 unsigned NumRegs = Regs.size();
930 SmallVector<SDValue, 8> Parts(NumRegs);
931 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
932 unsigned NumParts = RegCount[Value];
933
934 MVT RegisterVT = isABIMangled()
936 *DAG.getContext(), *CallConv, RegVTs[Value])
937 : RegVTs[Value];
938
939 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
940 ExtendKind = ISD::ZERO_EXTEND;
941
942 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
943 NumParts, RegisterVT, V, CallConv, ExtendKind);
944 Part += NumParts;
945 }
946
947 // Copy the parts into the registers.
948 SmallVector<SDValue, 8> Chains(NumRegs);
949 for (unsigned i = 0; i != NumRegs; ++i) {
950 SDValue Part;
951 if (!Flag) {
952 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
953 } else {
954 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
955 *Flag = Part.getValue(1);
956 }
957
958 Chains[i] = Part.getValue(0);
959 }
960
961 if (NumRegs == 1 || Flag)
962 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
963 // flagged to it. That is the CopyToReg nodes and the user are considered
964 // a single scheduling unit. If we create a TokenFactor and return it as
965 // chain, then the TokenFactor is both a predecessor (operand) of the
966 // user as well as a successor (the TF operands are flagged to the user).
967 // c1, f1 = CopyToReg
968 // c2, f2 = CopyToReg
969 // c3 = TokenFactor c1, c2
970 // ...
971 // = op c3, ..., f2
972 Chain = Chains[NumRegs-1];
973 else
974 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
975}
976
977void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
978 unsigned MatchingIdx, const SDLoc &dl,
979 SelectionDAG &DAG,
980 std::vector<SDValue> &Ops) const {
981 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
982
983 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
984 if (HasMatching)
985 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
986 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
987 // Put the register class of the virtual registers in the flag word. That
988 // way, later passes can recompute register class constraints for inline
989 // assembly as well as normal instructions.
990 // Don't do this for tied operands that can use the regclass information
991 // from the def.
993 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
994 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
995 }
996
997 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
998 Ops.push_back(Res);
999
1000 if (Code == InlineAsm::Kind_Clobber) {
1001 // Clobbers should always have a 1:1 mapping with registers, and may
1002 // reference registers that have illegal (e.g. vector) types. Hence, we
1003 // shouldn't try to apply any sort of splitting logic to them.
1004 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1005 "No 1:1 mapping from clobbers to regs?");
1007 (void)SP;
1008 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1009 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1010 assert(
1011 (Regs[I] != SP ||
1013 "If we clobbered the stack pointer, MFI should know about it.");
1014 }
1015 return;
1016 }
1017
1018 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1019 MVT RegisterVT = RegVTs[Value];
1020 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1021 RegisterVT);
1022 for (unsigned i = 0; i != NumRegs; ++i) {
1023 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1024 unsigned TheReg = Regs[Reg++];
1025 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1026 }
1027 }
1028}
1029
1033 unsigned I = 0;
1034 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1035 unsigned RegCount = std::get<0>(CountAndVT);
1036 MVT RegisterVT = std::get<1>(CountAndVT);
1037 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1038 for (unsigned E = I + RegCount; I != E; ++I)
1039 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1040 }
1041 return OutVec;
1042}
1043
1045 AssumptionCache *ac,
1046 const TargetLibraryInfo *li) {
1047 AA = aa;
1048 AC = ac;
1049 GFI = gfi;
1050 LibInfo = li;
1052 LPadToCallSiteMap.clear();
1054}
1055
1057 NodeMap.clear();
1058 UnusedArgNodeMap.clear();
1059 PendingLoads.clear();
1060 PendingExports.clear();
1061 PendingConstrainedFP.clear();
1062 PendingConstrainedFPStrict.clear();
1063 CurInst = nullptr;
1064 HasTailCall = false;
1065 SDNodeOrder = LowestSDNodeOrder;
1067}
1068
1070 DanglingDebugInfoMap.clear();
1071}
1072
1073// Update DAG root to include dependencies on Pending chains.
1074SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1075 SDValue Root = DAG.getRoot();
1076
1077 if (Pending.empty())
1078 return Root;
1079
1080 // Add current root to PendingChains, unless we already indirectly
1081 // depend on it.
1082 if (Root.getOpcode() != ISD::EntryToken) {
1083 unsigned i = 0, e = Pending.size();
1084 for (; i != e; ++i) {
1085 assert(Pending[i].getNode()->getNumOperands() > 1);
1086 if (Pending[i].getNode()->getOperand(0) == Root)
1087 break; // Don't add the root if we already indirectly depend on it.
1088 }
1089
1090 if (i == e)
1091 Pending.push_back(Root);
1092 }
1093
1094 if (Pending.size() == 1)
1095 Root = Pending[0];
1096 else
1097 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1098
1099 DAG.setRoot(Root);
1100 Pending.clear();
1101 return Root;
1102}
1103
1105 return updateRoot(PendingLoads);
1106}
1107
1109 // Chain up all pending constrained intrinsics together with all
1110 // pending loads, by simply appending them to PendingLoads and
1111 // then calling getMemoryRoot().
1112 PendingLoads.reserve(PendingLoads.size() +
1113 PendingConstrainedFP.size() +
1114 PendingConstrainedFPStrict.size());
1115 PendingLoads.append(PendingConstrainedFP.begin(),
1116 PendingConstrainedFP.end());
1117 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1118 PendingConstrainedFPStrict.end());
1119 PendingConstrainedFP.clear();
1120 PendingConstrainedFPStrict.clear();
1121 return getMemoryRoot();
1122}
1123
1125 // We need to emit pending fpexcept.strict constrained intrinsics,
1126 // so append them to the PendingExports list.
1127 PendingExports.append(PendingConstrainedFPStrict.begin(),
1128 PendingConstrainedFPStrict.end());
1129 PendingConstrainedFPStrict.clear();
1130 return updateRoot(PendingExports);
1131}
1132
1134 // Set up outgoing PHI node register values before emitting the terminator.
1135 if (I.isTerminator()) {
1136 HandlePHINodesInSuccessorBlocks(I.getParent());
1137 }
1138
1139 // Add SDDbgValue nodes for any var locs here. Do so before updating
1140 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1141 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1142 // Add SDDbgValue nodes for any var locs here. Do so before updating
1143 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1144 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1145 It != End; ++It) {
1146 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1147 dropDanglingDebugInfo(Var, It->Expr);
1148 SmallVector<Value *> Values(It->Values.location_ops());
1149 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1150 It->Values.hasArgList()))
1151 addDanglingDebugInfo(It, SDNodeOrder);
1152 }
1153 }
1154
1155 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1156 if (!isa<DbgInfoIntrinsic>(I))
1157 ++SDNodeOrder;
1158
1159 CurInst = &I;
1160
1161 // Set inserted listener only if required.
1162 bool NodeInserted = false;
1163 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1164 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1165 if (PCSectionsMD) {
1166 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1167 DAG, [&](SDNode *) { NodeInserted = true; });
1168 }
1169
1170 visit(I.getOpcode(), I);
1171
1172 if (!I.isTerminator() && !HasTailCall &&
1173 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1175
1176 // Handle metadata.
1177 if (PCSectionsMD) {
1178 auto It = NodeMap.find(&I);
1179 if (It != NodeMap.end()) {
1180 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1181 } else if (NodeInserted) {
1182 // This should not happen; if it does, don't let it go unnoticed so we can
1183 // fix it. Relevant visit*() function is probably missing a setValue().
1184 errs() << "warning: loosing !pcsections metadata ["
1185 << I.getModule()->getName() << "]\n";
1186 LLVM_DEBUG(I.dump());
1187 assert(false);
1188 }
1189 }
1190
1191 CurInst = nullptr;
1192}
1193
1194void SelectionDAGBuilder::visitPHI(const PHINode &) {
1195 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1196}
1197
1198void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1199 // Note: this doesn't use InstVisitor, because it has to work with
1200 // ConstantExpr's in addition to instructions.
1201 switch (Opcode) {
1202 default: llvm_unreachable("Unknown instruction type encountered!");
1203 // Build the switch statement using the Instruction.def file.
1204#define HANDLE_INST(NUM, OPCODE, CLASS) \
1205 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1206#include "llvm/IR/Instruction.def"
1207 }
1208}
1209
1211 DILocalVariable *Variable,
1212 DebugLoc DL, unsigned Order,
1213 RawLocationWrapper Values,
1215 if (!Values.hasArgList())
1216 return false;
1217 // For variadic dbg_values we will now insert an undef.
1218 // FIXME: We can potentially recover these!
1220 for (const Value *V : Values.location_ops()) {
1221 auto *Undef = UndefValue::get(V->getType());
1223 }
1224 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1225 /*IsIndirect=*/false, DL, Order,
1226 /*IsVariadic=*/true);
1227 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1228 return true;
1229}
1230
1232 unsigned Order) {
1234 DAG,
1235 const_cast<DILocalVariable *>(DAG.getFunctionVarLocs()
1236 ->getVariable(VarLoc->VariableID)
1237 .getVariable()),
1238 VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) {
1239 DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back(
1240 VarLoc, Order);
1241 }
1242}
1243
1245 unsigned Order) {
1246 // We treat variadic dbg_values differently at this stage.
1248 DAG, DI->getVariable(), DI->getDebugLoc(), Order,
1249 DI->getWrappedLocation(), DI->getExpression())) {
1250 // TODO: Dangling debug info will eventually either be resolved or produce
1251 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1252 // between the original dbg.value location and its resolved DBG_VALUE,
1253 // which we should ideally fill with an extra Undef DBG_VALUE.
1254 assert(DI->getNumVariableLocationOps() == 1 &&
1255 "DbgValueInst without an ArgList should have a single location "
1256 "operand.");
1257 DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order);
1258 }
1259}
1260
1262 const DIExpression *Expr) {
1263 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1264 DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs());
1265 DIExpression *DanglingExpr = DDI.getExpression();
1266 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1267 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI)
1268 << "\n");
1269 return true;
1270 }
1271 return false;
1272 };
1273
1274 for (auto &DDIMI : DanglingDebugInfoMap) {
1275 DanglingDebugInfoVector &DDIV = DDIMI.second;
1276
1277 // If debug info is to be dropped, run it through final checks to see
1278 // whether it can be salvaged.
1279 for (auto &DDI : DDIV)
1280 if (isMatchingDbgValue(DDI))
1282
1283 erase_if(DDIV, isMatchingDbgValue);
1284 }
1285}
1286
1287// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1288// generate the debug data structures now that we've seen its definition.
1290 SDValue Val) {
1291 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1292 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1293 return;
1294
1295 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1296 for (auto &DDI : DDIV) {
1297 DebugLoc DL = DDI.getDebugLoc();
1298 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1299 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1300 DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs());
1301 DIExpression *Expr = DDI.getExpression();
1303 "Expected inlined-at fields to agree");
1304 SDDbgValue *SDV;
1305 if (Val.getNode()) {
1306 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1307 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1308 // we couldn't resolve it directly when examining the DbgValue intrinsic
1309 // in the first place we should not be more successful here). Unless we
1310 // have some test case that prove this to be correct we should avoid
1311 // calling EmitFuncArgumentDbgValue here.
1312 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1313 FuncArgumentDbgValueKind::Value, Val)) {
1314 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI)
1315 << "\n");
1316 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1317 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1318 // inserted after the definition of Val when emitting the instructions
1319 // after ISel. An alternative could be to teach
1320 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1321 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1322 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1323 << ValSDNodeOrder << "\n");
1324 SDV = getDbgValue(Val, Variable, Expr, DL,
1325 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1326 DAG.AddDbgValue(SDV, false);
1327 } else
1328 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1329 << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n");
1330 } else {
1331 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n");
1332 auto Undef = UndefValue::get(V->getType());
1333 auto SDV =
1334 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1335 DAG.AddDbgValue(SDV, false);
1336 }
1337 }
1338 DDIV.clear();
1339}
1340
1342 // TODO: For the variadic implementation, instead of only checking the fail
1343 // state of `handleDebugValue`, we need know specifically which values were
1344 // invalid, so that we attempt to salvage only those values when processing
1345 // a DIArgList.
1346 Value *V = DDI.getVariableLocationOp(0);
1347 Value *OrigV = V;
1348 DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs());
1349 DIExpression *Expr = DDI.getExpression();
1350 DebugLoc DL = DDI.getDebugLoc();
1351 unsigned SDOrder = DDI.getSDNodeOrder();
1352
1353 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1354 // that DW_OP_stack_value is desired.
1355 bool StackValue = true;
1356
1357 // Can this Value can be encoded without any further work?
1358 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1359 return;
1360
1361 // Attempt to salvage back through as many instructions as possible. Bail if
1362 // a non-instruction is seen, such as a constant expression or global
1363 // variable. FIXME: Further work could recover those too.
1364 while (isa<Instruction>(V)) {
1365 Instruction &VAsInst = *cast<Instruction>(V);
1366 // Temporary "0", awaiting real implementation.
1368 SmallVector<Value *, 4> AdditionalValues;
1369 V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
1370 AdditionalValues);
1371 // If we cannot salvage any further, and haven't yet found a suitable debug
1372 // expression, bail out.
1373 if (!V)
1374 break;
1375
1376 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1377 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1378 // here for variadic dbg_values, remove that condition.
1379 if (!AdditionalValues.empty())
1380 break;
1381
1382 // New value and expr now represent this debuginfo.
1383 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1384
1385 // Some kind of simplification occurred: check whether the operand of the
1386 // salvaged debug expression can be encoded in this DAG.
1387 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1388 LLVM_DEBUG(
1389 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1390 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1391 return;
1392 }
1393 }
1394
1395 // This was the final opportunity to salvage this debug information, and it
1396 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1397 // any earlier variable location.
1398 assert(OrigV && "V shouldn't be null");
1399 auto *Undef = UndefValue::get(OrigV->getType());
1400 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1401 DAG.AddDbgValue(SDV, false);
1402 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << printDDI(DDI)
1403 << "\n");
1404}
1405
1407 DILocalVariable *Var,
1408 DIExpression *Expr, DebugLoc DbgLoc,
1409 unsigned Order, bool IsVariadic) {
1410 if (Values.empty())
1411 return true;
1412 SmallVector<SDDbgOperand> LocationOps;
1413 SmallVector<SDNode *> Dependencies;
1414 for (const Value *V : Values) {
1415 // Constant value.
1416 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1417 isa<ConstantPointerNull>(V)) {
1418 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1419 continue;
1420 }
1421
1422 // Look through IntToPtr constants.
1423 if (auto *CE = dyn_cast<ConstantExpr>(V))
1424 if (CE->getOpcode() == Instruction::IntToPtr) {
1425 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1426 continue;
1427 }
1428
1429 // If the Value is a frame index, we can create a FrameIndex debug value
1430 // without relying on the DAG at all.
1431 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1432 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1433 if (SI != FuncInfo.StaticAllocaMap.end()) {
1434 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1435 continue;
1436 }
1437 }
1438
1439 // Do not use getValue() in here; we don't want to generate code at
1440 // this point if it hasn't been done yet.
1441 SDValue N = NodeMap[V];
1442 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1443 N = UnusedArgNodeMap[V];
1444 if (N.getNode()) {
1445 // Only emit func arg dbg value for non-variadic dbg.values for now.
1446 if (!IsVariadic &&
1447 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1448 FuncArgumentDbgValueKind::Value, N))
1449 return true;
1450 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1451 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1452 // describe stack slot locations.
1453 //
1454 // Consider "int x = 0; int *px = &x;". There are two kinds of
1455 // interesting debug values here after optimization:
1456 //
1457 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1458 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1459 //
1460 // Both describe the direct values of their associated variables.
1461 Dependencies.push_back(N.getNode());
1462 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1463 continue;
1464 }
1465 LocationOps.emplace_back(
1466 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1467 continue;
1468 }
1469
1471 // Special rules apply for the first dbg.values of parameter variables in a
1472 // function. Identify them by the fact they reference Argument Values, that
1473 // they're parameters, and they are parameters of the current function. We
1474 // need to let them dangle until they get an SDNode.
1475 bool IsParamOfFunc =
1476 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1477 if (IsParamOfFunc)
1478 return false;
1479
1480 // The value is not used in this block yet (or it would have an SDNode).
1481 // We still want the value to appear for the user if possible -- if it has
1482 // an associated VReg, we can refer to that instead.
1483 auto VMI = FuncInfo.ValueMap.find(V);
1484 if (VMI != FuncInfo.ValueMap.end()) {
1485 unsigned Reg = VMI->second;
1486 // If this is a PHI node, it may be split up into several MI PHI nodes
1487 // (in FunctionLoweringInfo::set).
1488 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1489 V->getType(), std::nullopt);
1490 if (RFV.occupiesMultipleRegs()) {
1491 // FIXME: We could potentially support variadic dbg_values here.
1492 if (IsVariadic)
1493 return false;
1494 unsigned Offset = 0;
1495 unsigned BitsToDescribe = 0;
1496 if (auto VarSize = Var->getSizeInBits())
1497 BitsToDescribe = *VarSize;
1498 if (auto Fragment = Expr->getFragmentInfo())
1499 BitsToDescribe = Fragment->SizeInBits;
1500 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1501 // Bail out if all bits are described already.
1502 if (Offset >= BitsToDescribe)
1503 break;
1504 // TODO: handle scalable vectors.
1505 unsigned RegisterSize = RegAndSize.second;
1506 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1507 ? BitsToDescribe - Offset
1508 : RegisterSize;
1509 auto FragmentExpr = DIExpression::createFragmentExpression(
1510 Expr, Offset, FragmentSize);
1511 if (!FragmentExpr)
1512 continue;
1514 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
1515 DAG.AddDbgValue(SDV, false);
1516 Offset += RegisterSize;
1517 }
1518 return true;
1519 }
1520 // We can use simple vreg locations for variadic dbg_values as well.
1521 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1522 continue;
1523 }
1524 // We failed to create a SDDbgOperand for V.
1525 return false;
1526 }
1527
1528 // We have created a SDDbgOperand for each Value in Values.
1529 // Should use Order instead of SDNodeOrder?
1530 assert(!LocationOps.empty());
1531 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1532 /*IsIndirect=*/false, DbgLoc,
1533 SDNodeOrder, IsVariadic);
1534 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1535 return true;
1536}
1537
1539 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1540 for (auto &Pair : DanglingDebugInfoMap)
1541 for (auto &DDI : Pair.second)
1544}
1545
1546/// getCopyFromRegs - If there was virtual register allocated for the value V
1547/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1550 SDValue Result;
1551
1552 if (It != FuncInfo.ValueMap.end()) {
1553 Register InReg = It->second;
1554
1556 DAG.getDataLayout(), InReg, Ty,
1557 std::nullopt); // This is not an ABI copy.
1558 SDValue Chain = DAG.getEntryNode();
1559 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1560 V);
1561 resolveDanglingDebugInfo(V, Result);
1562 }
1563
1564 return Result;
1565}
1566
1567/// getValue - Return an SDValue for the given Value.
1569 // If we already have an SDValue for this value, use it. It's important
1570 // to do this first, so that we don't create a CopyFromReg if we already
1571 // have a regular SDValue.
1572 SDValue &N = NodeMap[V];
1573 if (N.getNode()) return N;
1574
1575 // If there's a virtual register allocated and initialized for this
1576 // value, use it.
1577 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1578 return copyFromReg;
1579
1580 // Otherwise create a new SDValue and remember it.
1581 SDValue Val = getValueImpl(V);
1582 NodeMap[V] = Val;
1584 return Val;
1585}
1586
1587/// getNonRegisterValue - Return an SDValue for the given Value, but
1588/// don't look in FuncInfo.ValueMap for a virtual register.
1590 // If we already have an SDValue for this value, use it.
1591 SDValue &N = NodeMap[V];
1592 if (N.getNode()) {
1593 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1594 // Remove the debug location from the node as the node is about to be used
1595 // in a location which may differ from the original debug location. This
1596 // is relevant to Constant and ConstantFP nodes because they can appear
1597 // as constant expressions inside PHI nodes.
1598 N->setDebugLoc(DebugLoc());
1599 }
1600 return N;
1601 }
1602
1603 // Otherwise create a new SDValue and remember it.
1604 SDValue Val = getValueImpl(V);
1605 NodeMap[V] = Val;
1607 return Val;
1608}
1609
1610/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1611/// Create an SDValue for the given value.
1614
1615 if (const Constant *C = dyn_cast<Constant>(V)) {
1616 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1617
1618 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1619 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1620
1621 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1622 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1623
1624 if (isa<ConstantPointerNull>(C)) {
1625 unsigned AS = V->getType()->getPointerAddressSpace();
1626 return DAG.getConstant(0, getCurSDLoc(),
1627 TLI.getPointerTy(DAG.getDataLayout(), AS));
1628 }
1629
1630 if (match(C, m_VScale()))
1631 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1632
1633 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1634 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1635
1636 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1637 return DAG.getUNDEF(VT);
1638
1639 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1640 visit(CE->getOpcode(), *CE);
1641 SDValue N1 = NodeMap[V];
1642 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1643 return N1;
1644 }
1645
1646 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1648 for (const Use &U : C->operands()) {
1649 SDNode *Val = getValue(U).getNode();
1650 // If the operand is an empty aggregate, there are no values.
1651 if (!Val) continue;
1652 // Add each leaf value from the operand to the Constants list
1653 // to form a flattened list of all the values.
1654 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1655 Constants.push_back(SDValue(Val, i));
1656 }
1657
1659 }
1660
1661 if (const ConstantDataSequential *CDS =
1662 dyn_cast<ConstantDataSequential>(C)) {
1664 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1665 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1666 // Add each leaf value from the operand to the Constants list
1667 // to form a flattened list of all the values.
1668 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1669 Ops.push_back(SDValue(Val, i));
1670 }
1671
1672 if (isa<ArrayType>(CDS->getType()))
1673 return DAG.getMergeValues(Ops, getCurSDLoc());
1674 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1675 }
1676
1677 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1678 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1679 "Unknown struct or array constant!");
1680
1681 SmallVector<EVT, 4> ValueVTs;
1682 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1683 unsigned NumElts = ValueVTs.size();
1684 if (NumElts == 0)
1685 return SDValue(); // empty struct
1687 for (unsigned i = 0; i != NumElts; ++i) {
1688 EVT EltVT = ValueVTs[i];
1689 if (isa<UndefValue>(C))
1690 Constants[i] = DAG.getUNDEF(EltVT);
1691 else if (EltVT.isFloatingPoint())
1692 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1693 else
1694 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1695 }
1696
1698 }
1699
1700 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1701 return DAG.getBlockAddress(BA, VT);
1702
1703 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1704 return getValue(Equiv->getGlobalValue());
1705
1706 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1707 return getValue(NC->getGlobalValue());
1708
1709 VectorType *VecTy = cast<VectorType>(V->getType());
1710
1711 // Now that we know the number and type of the elements, get that number of
1712 // elements into the Ops array based on what kind of constant it is.
1713 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1715 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1716 for (unsigned i = 0; i != NumElements; ++i)
1717 Ops.push_back(getValue(CV->getOperand(i)));
1718
1719 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1720 }
1721
1722 if (isa<ConstantAggregateZero>(C)) {
1723 EVT EltVT =
1725
1726 SDValue Op;
1727 if (EltVT.isFloatingPoint())
1728 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1729 else
1730 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1731
1732 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1733 }
1734
1735 llvm_unreachable("Unknown vector constant");
1736 }
1737
1738 // If this is a static alloca, generate it as the frameindex instead of
1739 // computation.
1740 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1742 FuncInfo.StaticAllocaMap.find(AI);
1743 if (SI != FuncInfo.StaticAllocaMap.end())
1744 return DAG.getFrameIndex(
1745 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1746 }
1747
1748 // If this is an instruction which fast-isel has deferred, select it now.
1749 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1750 Register InReg = FuncInfo.InitializeRegForValue(Inst);
1751
1752 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1753 Inst->getType(), std::nullopt);
1754 SDValue Chain = DAG.getEntryNode();
1755 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1756 }
1757
1758 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1759 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1760
1761 if (const auto *BB = dyn_cast<BasicBlock>(V))
1762 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
1763
1764 llvm_unreachable("Can't get register for value!");
1765}
1766
1767void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1768 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1769 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1770 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1771 bool IsSEH = isAsynchronousEHPersonality(Pers);
1772 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1773 if (!IsSEH)
1774 CatchPadMBB->setIsEHScopeEntry();
1775 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1776 if (IsMSVCCXX || IsCoreCLR)
1777 CatchPadMBB->setIsEHFuncletEntry();
1778}
1779
1780void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1781 // Update machine-CFG edge.
1782 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1783 FuncInfo.MBB->addSuccessor(TargetMBB);
1784 TargetMBB->setIsEHCatchretTarget(true);
1786
1787 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1788 bool IsSEH = isAsynchronousEHPersonality(Pers);
1789 if (IsSEH) {
1790 // If this is not a fall-through branch or optimizations are switched off,
1791 // emit the branch.
1792 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1795 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1796 return;
1797 }
1798
1799 // Figure out the funclet membership for the catchret's successor.
1800 // This will be used by the FuncletLayout pass to determine how to order the
1801 // BB's.
1802 // A 'catchret' returns to the outer scope's color.
1803 Value *ParentPad = I.getCatchSwitchParentPad();
1804 const BasicBlock *SuccessorColor;
1805 if (isa<ConstantTokenNone>(ParentPad))
1806 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1807 else
1808 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1809 assert(SuccessorColor && "No parent funclet for catchret!");
1810 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1811 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1812
1813 // Create the terminator node.
1815 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1816 DAG.getBasicBlock(SuccessorColorMBB));
1817 DAG.setRoot(Ret);
1818}
1819
1820void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1821 // Don't emit any special code for the cleanuppad instruction. It just marks
1822 // the start of an EH scope/funclet.
1823 FuncInfo.MBB->setIsEHScopeEntry();
1824 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1825 if (Pers != EHPersonality::Wasm_CXX) {
1826 FuncInfo.MBB->setIsEHFuncletEntry();
1827 FuncInfo.MBB->setIsCleanupFuncletEntry();
1828 }
1829}
1830
1831// In wasm EH, even though a catchpad may not catch an exception if a tag does
1832// not match, it is OK to add only the first unwind destination catchpad to the
1833// successors, because there will be at least one invoke instruction within the
1834// catch scope that points to the next unwind destination, if one exists, so
1835// CFGSort cannot mess up with BB sorting order.
1836// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1837// call within them, and catchpads only consisting of 'catch (...)' have a
1838// '__cxa_end_catch' call within them, both of which generate invokes in case
1839// the next unwind destination exists, i.e., the next unwind destination is not
1840// the caller.)
1841//
1842// Having at most one EH pad successor is also simpler and helps later
1843// transformations.
1844//
1845// For example,
1846// current:
1847// invoke void @foo to ... unwind label %catch.dispatch
1848// catch.dispatch:
1849// %0 = catchswitch within ... [label %catch.start] unwind label %next
1850// catch.start:
1851// ...
1852// ... in this BB or some other child BB dominated by this BB there will be an
1853// invoke that points to 'next' BB as an unwind destination
1854//
1855// next: ; We don't need to add this to 'current' BB's successor
1856// ...
1858 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1859 BranchProbability Prob,
1860 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1861 &UnwindDests) {
1862 while (EHPadBB) {
1863 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1864 if (isa<CleanupPadInst>(Pad)) {
1865 // Stop on cleanup pads.
1866 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1867 UnwindDests.back().first->setIsEHScopeEntry();
1868 break;
1869 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1870 // Add the catchpad handlers to the possible destinations. We don't
1871 // continue to the unwind destination of the catchswitch for wasm.
1872 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1873 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1874 UnwindDests.back().first->setIsEHScopeEntry();
1875 }
1876 break;
1877 } else {
1878 continue;
1879 }
1880 }
1881}
1882
1883/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1884/// many places it could ultimately go. In the IR, we have a single unwind
1885/// destination, but in the machine CFG, we enumerate all the possible blocks.
1886/// This function skips over imaginary basic blocks that hold catchswitch
1887/// instructions, and finds all the "real" machine
1888/// basic block destinations. As those destinations may not be successors of
1889/// EHPadBB, here we also calculate the edge probability to those destinations.
1890/// The passed-in Prob is the edge probability to EHPadBB.
1892 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1893 BranchProbability Prob,
1894 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1895 &UnwindDests) {
1896 EHPersonality Personality =
1897 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1898 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1899 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1900 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1901 bool IsSEH = isAsynchronousEHPersonality(Personality);
1902
1903 if (IsWasmCXX) {
1904 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1905 assert(UnwindDests.size() <= 1 &&
1906 "There should be at most one unwind destination for wasm");
1907 return;
1908 }
1909
1910 while (EHPadBB) {
1911 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1912 BasicBlock *NewEHPadBB = nullptr;
1913 if (isa<LandingPadInst>(Pad)) {
1914 // Stop on landingpads. They are not funclets.
1915 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1916 break;
1917 } else if (isa<CleanupPadInst>(Pad)) {
1918 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1919 // personalities.
1920 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1921 UnwindDests.back().first->setIsEHScopeEntry();
1922 UnwindDests.back().first->setIsEHFuncletEntry();
1923 break;
1924 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1925 // Add the catchpad handlers to the possible destinations.
1926 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1927 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1928 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1929 if (IsMSVCCXX || IsCoreCLR)
1930 UnwindDests.back().first->setIsEHFuncletEntry();
1931 if (!IsSEH)
1932 UnwindDests.back().first->setIsEHScopeEntry();
1933 }
1934 NewEHPadBB = CatchSwitch->getUnwindDest();
1935 } else {
1936 continue;
1937 }
1938
1940 if (BPI && NewEHPadBB)
1941 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1942 EHPadBB = NewEHPadBB;
1943 }
1944}
1945
1946void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1947 // Update successor info.
1949 auto UnwindDest = I.getUnwindDest();
1951 BranchProbability UnwindDestProb =
1952 (BPI && UnwindDest)
1953 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1955 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1956 for (auto &UnwindDest : UnwindDests) {
1957 UnwindDest.first->setIsEHPad();
1958 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1959 }
1960 FuncInfo.MBB->normalizeSuccProbs();
1961
1962 // Create the terminator node.
1963 SDValue Ret =
1965 DAG.setRoot(Ret);
1966}
1967
1968void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1969 report_fatal_error("visitCatchSwitch not yet implemented!");
1970}
1971
1972void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1974 auto &DL = DAG.getDataLayout();
1975 SDValue Chain = getControlRoot();
1978
1979 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1980 // lower
1981 //
1982 // %val = call <ty> @llvm.experimental.deoptimize()
1983 // ret <ty> %val
1984 //
1985 // differently.
1986 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1988 return;
1989 }
1990
1991 if (!FuncInfo.CanLowerReturn) {
1992 unsigned DemoteReg = FuncInfo.DemoteRegister;
1993 const Function *F = I.getParent()->getParent();
1994
1995 // Emit a store of the return value through the virtual register.
1996 // Leave Outs empty so that LowerReturn won't try to load return
1997 // registers the usual way.
1998 SmallVector<EVT, 1> PtrValueVTs;
1999 ComputeValueVTs(TLI, DL,
2000 F->getReturnType()->getPointerTo(
2002 PtrValueVTs);
2003
2004 SDValue RetPtr =
2005 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
2006 SDValue RetOp = getValue(I.getOperand(0));
2007
2008 SmallVector<EVT, 4> ValueVTs, MemVTs;
2010 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2011 &Offsets);
2012 unsigned NumValues = ValueVTs.size();
2013
2014 SmallVector<SDValue, 4> Chains(NumValues);
2015 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2016 for (unsigned i = 0; i != NumValues; ++i) {
2017 // An aggregate return value cannot wrap around the address space, so
2018 // offsets to its parts don't wrap either.
2020 TypeSize::Fixed(Offsets[i]));
2021
2022 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2023 if (MemVTs[i] != ValueVTs[i])
2024 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2025 Chains[i] = DAG.getStore(
2026 Chain, getCurSDLoc(), Val,
2027 // FIXME: better loc info would be nice.
2029 commonAlignment(BaseAlign, Offsets[i]));
2030 }
2031
2033 MVT::Other, Chains);
2034 } else if (I.getNumOperands() != 0) {
2035 SmallVector<EVT, 4> ValueVTs;
2036 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2037 unsigned NumValues = ValueVTs.size();
2038 if (NumValues) {
2039 SDValue RetOp = getValue(I.getOperand(0));
2040
2041 const Function *F = I.getParent()->getParent();
2042
2043 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2044 I.getOperand(0)->getType(), F->getCallingConv(),
2045 /*IsVarArg*/ false, DL);
2046
2047 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2048 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2049 ExtendKind = ISD::SIGN_EXTEND;
2050 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2051 ExtendKind = ISD::ZERO_EXTEND;
2052
2053 LLVMContext &Context = F->getContext();
2054 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2055
2056 for (unsigned j = 0; j != NumValues; ++j) {
2057 EVT VT = ValueVTs[j];
2058
2059 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2060 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2061
2062 CallingConv::ID CC = F->getCallingConv();
2063
2064 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2065 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2066 SmallVector<SDValue, 4> Parts(NumParts);
2068 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2069 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2070
2071 // 'inreg' on function refers to return value
2073 if (RetInReg)
2074 Flags.setInReg();
2075
2076 if (I.getOperand(0)->getType()->isPointerTy()) {
2077 Flags.setPointer();
2078 Flags.setPointerAddrSpace(
2079 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2080 }
2081
2082 if (NeedsRegBlock) {
2083 Flags.setInConsecutiveRegs();
2084 if (j == NumValues - 1)
2085 Flags.setInConsecutiveRegsLast();
2086 }
2087
2088 // Propagate extension type if any
2089 if (ExtendKind == ISD::SIGN_EXTEND)
2090 Flags.setSExt();
2091 else if (ExtendKind == ISD::ZERO_EXTEND)
2092 Flags.setZExt();
2093
2094 for (unsigned i = 0; i < NumParts; ++i) {
2095 Outs.push_back(ISD::OutputArg(Flags,
2096 Parts[i].getValueType().getSimpleVT(),
2097 VT, /*isfixed=*/true, 0, 0));
2098 OutVals.push_back(Parts[i]);
2099 }
2100 }
2101 }
2102 }
2103
2104 // Push in swifterror virtual register as the last element of Outs. This makes
2105 // sure swifterror virtual register will be returned in the swifterror
2106 // physical register.
2107 const Function *F = I.getParent()->getParent();
2108 if (TLI.supportSwiftError() &&
2109 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2110 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2112 Flags.setSwiftError();
2114 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2115 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2116 // Create SDNode for the swifterror virtual register.
2117 OutVals.push_back(
2120 EVT(TLI.getPointerTy(DL))));
2121 }
2122
2123 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2124 CallingConv::ID CallConv =
2127 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2128
2129 // Verify that the target's LowerReturn behaved as expected.
2130 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2131 "LowerReturn didn't return a valid chain!");
2132
2133 // Update the DAG with the new chain value resulting from return lowering.
2134 DAG.setRoot(Chain);
2135}
2136
2137/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2138/// created for it, emit nodes to copy the value into the virtual
2139/// registers.
2141 // Skip empty types
2142 if (V->getType()->isEmptyTy())
2143 return;
2144
2146 if (VMI != FuncInfo.ValueMap.end()) {
2147 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2148 "Unused value assigned virtual registers!");
2149 CopyValueToVirtualRegister(V, VMI->second);
2150 }
2151}
2152
2153/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2154/// the current basic block, add it to ValueMap now so that we'll get a
2155/// CopyTo/FromReg.
2157 // No need to export constants.
2158 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2159
2160 // Already exported?
2161 if (FuncInfo.isExportedInst(V)) return;
2162
2163 Register Reg = FuncInfo.InitializeRegForValue(V);
2165}
2166
2168 const BasicBlock *FromBB) {
2169 // The operands of the setcc have to be in this block. We don't know
2170 // how to export them from some other block.
2171 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2172 // Can export from current BB.
2173 if (VI->getParent() == FromBB)
2174 return true;
2175
2176 // Is already exported, noop.
2177 return FuncInfo.isExportedInst(V);
2178 }
2179
2180 // If this is an argument, we can export it if the BB is the entry block or
2181 // if it is already exported.
2182 if (isa<Argument>(V)) {
2183 if (FromBB->isEntryBlock())
2184 return true;
2185
2186 // Otherwise, can only export this if it is already exported.
2187 return FuncInfo.isExportedInst(V);
2188 }
2189
2190 // Otherwise, constants can always be exported.
2191 return true;
2192}
2193
2194/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2196SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2197 const MachineBasicBlock *Dst) const {
2199 const BasicBlock *SrcBB = Src->getBasicBlock();
2200 const BasicBlock *DstBB = Dst->getBasicBlock();
2201 if (!BPI) {
2202 // If BPI is not available, set the default probability as 1 / N, where N is
2203 // the number of successors.
2204 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2205 return BranchProbability(1, SuccSize);
2206 }
2207 return BPI->getEdgeProbability(SrcBB, DstBB);
2208}
2209
2210void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2211 MachineBasicBlock *Dst,
2212 BranchProbability Prob) {
2213 if (!FuncInfo.BPI)
2214 Src->addSuccessorWithoutProb(Dst);
2215 else {
2216 if (Prob.isUnknown())
2217 Prob = getEdgeProbability(Src, Dst);
2218 Src->addSuccessor(Dst, Prob);
2219 }
2220}
2221
2222static bool InBlock(const Value *V, const BasicBlock *BB) {
2223 if (const Instruction *I = dyn_cast<Instruction>(V))
2224 return I->getParent() == BB;
2225 return true;
2226}
2227
2228/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2229/// This function emits a branch and is used at the leaves of an OR or an
2230/// AND operator tree.
2231void
2234 MachineBasicBlock *FBB,
2235 MachineBasicBlock *CurBB,
2236 MachineBasicBlock *SwitchBB,
2237 BranchProbability TProb,
2238 BranchProbability FProb,
2239 bool InvertCond) {
2240 const BasicBlock *BB = CurBB->getBasicBlock();
2241
2242 // If the leaf of the tree is a comparison, merge the condition into
2243 // the caseblock.
2244 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2245 // The operands of the cmp have to be in this block. We don't know
2246 // how to export them from some other block. If this is the first block
2247 // of the sequence, no exporting is needed.
2248 if (CurBB == SwitchBB ||
2249 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2250 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2251 ISD::CondCode Condition;
2252 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2253 ICmpInst::Predicate Pred =
2254 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2255 Condition = getICmpCondCode(Pred);
2256 } else {
2257 const FCmpInst *FC = cast<FCmpInst>(Cond);
2258 FCmpInst::Predicate Pred =
2259 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2260 Condition = getFCmpCondCode(Pred);
2261 if (TM.Options.NoNaNsFPMath)
2262 Condition = getFCmpCodeWithoutNaN(Condition);
2263 }
2264
2265 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2266 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2267 SL->SwitchCases.push_back(CB);
2268 return;
2269 }
2270 }
2271
2272 // Create a CaseBlock record representing this branch.
2273 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2275 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2276 SL->SwitchCases.push_back(CB);
2277}
2278
2281 MachineBasicBlock *FBB,
2282 MachineBasicBlock *CurBB,
2283 MachineBasicBlock *SwitchBB,
2285 BranchProbability TProb,
2286 BranchProbability FProb,
2287 bool InvertCond) {
2288 // Skip over not part of the tree and remember to invert op and operands at
2289 // next level.
2290 Value *NotCond;
2291 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2292 InBlock(NotCond, CurBB->getBasicBlock())) {
2293 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2294 !InvertCond);
2295 return;
2296 }
2297
2298 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2299 const Value *BOpOp0, *BOpOp1;
2300 // Compute the effective opcode for Cond, taking into account whether it needs
2301 // to be inverted, e.g.
2302 // and (not (or A, B)), C
2303 // gets lowered as
2304 // and (and (not A, not B), C)
2306 if (BOp) {
2307 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2308 ? Instruction::And
2309 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2310 ? Instruction::Or
2312 if (InvertCond) {
2313 if (BOpc == Instruction::And)
2314 BOpc = Instruction::Or;
2315 else if (BOpc == Instruction::Or)
2316 BOpc = Instruction::And;
2317 }
2318 }
2319
2320 // If this node is not part of the or/and tree, emit it as a branch.
2321 // Note that all nodes in the tree should have same opcode.
2322 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2323 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2324 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2325 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2326 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2327 TProb, FProb, InvertCond);
2328 return;
2329 }
2330
2331 // Create TmpBB after CurBB.
2332 MachineFunction::iterator BBI(CurBB);
2335 CurBB->getParent()->insert(++BBI, TmpBB);
2336
2337 if (Opc == Instruction::Or) {
2338 // Codegen X | Y as:
2339 // BB1:
2340 // jmp_if_X TBB
2341 // jmp TmpBB
2342 // TmpBB:
2343 // jmp_if_Y TBB
2344 // jmp FBB
2345 //
2346
2347 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2348 // The requirement is that
2349 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2350 // = TrueProb for original BB.
2351 // Assuming the original probabilities are A and B, one choice is to set
2352 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2353 // A/(1+B) and 2B/(1+B). This choice assumes that
2354 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2355 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2356 // TmpBB, but the math is more complicated.
2357
2358 auto NewTrueProb = TProb / 2;
2359 auto NewFalseProb = TProb / 2 + FProb;
2360 // Emit the LHS condition.
2361 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2362 NewFalseProb, InvertCond);
2363
2364 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2365 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2366 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2367 // Emit the RHS condition into TmpBB.
2368 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2369 Probs[1], InvertCond);
2370 } else {
2371 assert(Opc == Instruction::And && "Unknown merge op!");
2372 // Codegen X & Y as:
2373 // BB1:
2374 // jmp_if_X TmpBB
2375 // jmp FBB
2376 // TmpBB:
2377 // jmp_if_Y TBB
2378 // jmp FBB
2379 //
2380 // This requires creation of TmpBB after CurBB.
2381
2382 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2383 // The requirement is that
2384 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2385 // = FalseProb for original BB.
2386 // Assuming the original probabilities are A and B, one choice is to set
2387 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2388 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2389 // TrueProb for BB1 * FalseProb for TmpBB.
2390
2391 auto NewTrueProb = TProb + FProb / 2;
2392 auto NewFalseProb = FProb / 2;
2393 // Emit the LHS condition.
2394 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2395 NewFalseProb, InvertCond);
2396
2397 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2398 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2399 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2400 // Emit the RHS condition into TmpBB.
2401 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2402 Probs[1], InvertCond);
2403 }
2404}
2405
2406/// If the set of cases should be emitted as a series of branches, return true.
2407/// If we should emit this as a bunch of and/or'd together conditions, return
2408/// false.
2409bool
2410SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2411 if (Cases.size() != 2) return true;
2412
2413 // If this is two comparisons of the same values or'd or and'd together, they
2414 // will get folded into a single comparison, so don't emit two blocks.
2415 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2416 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2417 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2418 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2419 return false;
2420 }
2421
2422 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2423 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2424 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2425 Cases[0].CC == Cases[1].CC &&
2426 isa<Constant>(Cases[0].CmpRHS) &&
2427 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2428 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2429 return false;
2430 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2431 return false;
2432 }
2433
2434 return true;
2435}
2436
2437void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2438 MachineBasicBlock *BrMBB = FuncInfo.MBB;
2439
2440 // Update machine-CFG edges.
2441 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2442
2443 if (I.isUnconditional()) {
2444 // Update machine-CFG edges.
2445 BrMBB->addSuccessor(Succ0MBB);
2446
2447 // If this is not a fall-through branch or optimizations are switched off,
2448 // emit the branch.
2449 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
2452 DAG.getBasicBlock(Succ0MBB)));
2453
2454 return;
2455 }
2456
2457 // If this condition is one of the special cases we handle, do special stuff
2458 // now.
2459 const Value *CondVal = I.getCondition();
2460 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2461
2462 // If this is a series of conditions that are or'd or and'd together, emit
2463 // this as a sequence of branches instead of setcc's with and/or operations.
2464 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2465 // unpredictable branches, and vector extracts because those jumps are likely
2466 // expensive for any target), this should improve performance.
2467 // For example, instead of something like:
2468 // cmp A, B
2469 // C = seteq
2470 // cmp D, E
2471 // F = setle
2472 // or C, F
2473 // jnz foo
2474 // Emit:
2475 // cmp A, B
2476 // je foo
2477 // cmp D, E
2478 // jle foo
2479 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2480 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2481 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2482 Value *Vec;
2483 const Value *BOp0, *BOp1;
2485 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2486 Opcode = Instruction::And;
2487 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2488 Opcode = Instruction::Or;
2489
2490 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2491 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
2492 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2493 getEdgeProbability(BrMBB, Succ0MBB),
2494 getEdgeProbability(BrMBB, Succ1MBB),
2495 /*InvertCond=*/false);
2496 // If the compares in later blocks need to use values not currently
2497 // exported from this block, export them now. This block should always
2498 // be the first entry.
2499 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2500
2501 // Allow some cases to be rejected.
2502 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2503 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2504 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2505 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2506 }
2507
2508 // Emit the branch for this block.
2509 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2510 SL->SwitchCases.erase(SL->SwitchCases.begin());
2511 return;
2512 }
2513
2514 // Okay, we decided not to do this, remove any inserted MBB's and clear
2515 // SwitchCases.
2516 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2517 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2518
2519 SL->SwitchCases.clear();
2520 }
2521 }
2522
2523 // Create a CaseBlock record representing this branch.
2525 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2526
2527 // Use visitSwitchCase to actually insert the fast branch sequence for this
2528 // cond branch.
2529 visitSwitchCase(CB, BrMBB);
2530}
2531
2532/// visitSwitchCase - Emits the necessary code to represent a single node in
2533/// the binary search tree resulting from lowering a switch instruction.
2535 MachineBasicBlock *SwitchBB) {
2536 SDValue Cond;
2537 SDValue CondLHS = getValue(CB.CmpLHS);
2538 SDLoc dl = CB.DL;
2539
2540 if (CB.CC == ISD::SETTRUE) {
2541 // Branch or fall through to TrueBB.
2542 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2543 SwitchBB->normalizeSuccProbs();
2544 if (CB.TrueBB != NextBlock(SwitchBB)) {
2546 DAG.getBasicBlock(CB.TrueBB)));
2547 }
2548 return;
2549 }
2550
2551 auto &TLI = DAG.getTargetLoweringInfo();
2552 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2553
2554 // Build the setcc now.
2555 if (!CB.CmpMHS) {
2556 // Fold "(X == true)" to X and "(X == false)" to !X to
2557 // handle common cases produced by branch lowering.
2558 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2559 CB.CC == ISD::SETEQ)
2560 Cond = CondLHS;
2561 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2562 CB.CC == ISD::SETEQ) {
2563 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2564 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2565 } else {
2566 SDValue CondRHS = getValue(CB.CmpRHS);
2567
2568 // If a pointer's DAG type is larger than its memory type then the DAG
2569 // values are zero-extended. This breaks signed comparisons so truncate
2570 // back to the underlying type before doing the compare.
2571 if (CondLHS.getValueType() != MemVT) {
2572 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2573 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2574 }
2575 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2576 }
2577 } else {
2578 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2579
2580 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2581 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2582
2583 SDValue CmpOp = getValue(CB.CmpMHS);
2584 EVT VT = CmpOp.getValueType();
2585
2586 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2587 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2588 ISD::SETLE);
2589 } else {
2590 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2591 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2592 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2593 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2594 }
2595 }
2596
2597 // Update successor info
2598 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2599 // TrueBB and FalseBB are always different unless the incoming IR is
2600 // degenerate. This only happens when running llc on weird IR.
2601 if (CB.TrueBB != CB.FalseBB)
2602 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2603 SwitchBB->normalizeSuccProbs();
2604
2605 // If the lhs block is the next block, invert the condition so that we can
2606 // fall through to the lhs instead of the rhs block.
2607 if (CB.TrueBB == NextBlock(SwitchBB)) {
2608 std::swap(CB.TrueBB, CB.FalseBB);
2609 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2610 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2611 }
2612
2613 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2616
2617 setValue(CurInst, BrCond);
2618
2619 // Insert the false branch. Do this even if it's a fall through branch,
2620 // this makes it easier to do DAG optimizations which require inverting
2621 // the branch condition.
2622 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2624
2625 DAG.setRoot(BrCond);
2626}
2627
2628/// visitJumpTable - Emit JumpTable node in the current MBB
2630 // Emit the code for the jump table
2631 assert(JT.Reg != -1U && "Should lower JT Header first!");
2634 JT.Reg, PTy);
2635 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2636 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2637 MVT::Other, Index.getValue(1),
2638 Table, Index);
2639 DAG.setRoot(BrJumpTable);
2640}
2641
2642/// visitJumpTableHeader - This function emits necessary code to produce index
2643/// in the JumpTable from switch case.
2645 JumpTableHeader &JTH,
2646 MachineBasicBlock *SwitchBB) {
2647 SDLoc dl = getCurSDLoc();
2648
2649 // Subtract the lowest switch case value from the value being switched on.
2650 SDValue SwitchOp = getValue(JTH.SValue);
2651 EVT VT = SwitchOp.getValueType();
2652 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2653 DAG.getConstant(JTH.First, dl, VT));
2654
2655 // The SDNode we just created, which holds the value being switched on minus
2656 // the smallest case value, needs to be copied to a virtual register so it
2657 // can be used as an index into the jump table in a subsequent basic block.
2658 // This value may be smaller or larger than the target's pointer type, and
2659 // therefore require extension or truncating.
2661 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2662
2663 unsigned JumpTableReg =
2664 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2665 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2666 JumpTableReg, SwitchOp);
2667 JT.Reg = JumpTableReg;
2668
2669 if (!JTH.FallthroughUnreachable) {
2670 // Emit the range check for the jump table, and branch to the default block
2671 // for the switch statement if the value being switched on exceeds the
2672 // largest case in the switch.
2673 SDValue CMP = DAG.getSetCC(
2675 Sub.getValueType()),
2676 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2677
2678 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2679 MVT::Other, CopyTo, CMP,
2680 DAG.getBasicBlock(JT.Default));
2681
2682 // Avoid emitting unnecessary branches to the next block.
2683 if (JT.MBB != NextBlock(SwitchBB))
2684 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2685 DAG.getBasicBlock(JT.MBB));
2686
2687 DAG.setRoot(BrCond);
2688 } else {
2689 // Avoid emitting unnecessary branches to the next block.
2690 if (JT.MBB != NextBlock(SwitchBB))
2691 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2692 DAG.getBasicBlock(JT.MBB)));
2693 else
2694 DAG.setRoot(CopyTo);
2695 }
2696}
2697
2698/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2699/// variable if there exists one.
2701 SDValue &Chain) {
2702 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2703 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2704 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2708 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2709 if (Global) {
2710 MachinePointerInfo MPInfo(Global);
2714 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
2715 DAG.setNodeMemRefs(Node, {MemRef});
2716 }
2717 if (PtrTy != PtrMemTy)
2718 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2719 return SDValue(Node, 0);
2720}
2721
2722/// Codegen a new tail for a stack protector check ParentMBB which has had its
2723/// tail spliced into a stack protector check success bb.
2724///
2725/// For a high level explanation of how this fits into the stack protector
2726/// generation see the comment on the declaration of class
2727/// StackProtectorDescriptor.
2729 MachineBasicBlock *ParentBB) {
2730
2731 // First create the loads to the guard/stack slot for the comparison.
2733 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2734 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2735
2736 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2737 int FI = MFI.getStackProtectorIndex();
2738
2739 SDValue Guard;
2740 SDLoc dl = getCurSDLoc();
2741 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2742 const Module &M = *ParentBB->getParent()->getFunction().getParent();
2743 Align Align =
2745
2746 // Generate code to load the content of the guard slot.
2747 SDValue GuardVal = DAG.getLoad(
2748 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2751
2752 if (TLI.useStackGuardXorFP())
2753 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2754
2755 // Retrieve guard check function, nullptr if instrumentation is inlined.
2756 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2757 // The target provides a guard check function to validate the guard value.
2758 // Generate a call to that function with the content of the guard slot as
2759 // argument.
2760 FunctionType *FnTy = GuardCheckFn->getFunctionType();
2761 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2762
2765 Entry.Node = GuardVal;
2766 Entry.Ty = FnTy->getParamType(0);
2767 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
2768 Entry.IsInReg = true;
2769 Args.push_back(Entry);
2770
2774 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2775 getValue(GuardCheckFn), std::move(Args));
2776
2777 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2778 DAG.setRoot(Result.second);
2779 return;
2780 }
2781
2782 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2783 // Otherwise, emit a volatile load to retrieve the stack guard value.
2784 SDValue Chain = DAG.getEntryNode();
2785 if (TLI.useLoadStackGuardNode()) {
2786 Guard = getLoadStackGuard(DAG, dl, Chain);
2787 } else {
2788 const Value *IRGuard = TLI.getSDagStackGuard(M);
2789 SDValue GuardPtr = getValue(IRGuard);
2790
2791 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2792 MachinePointerInfo(IRGuard, 0), Align,
2794 }
2795
2796 // Perform the comparison via a getsetcc.
2798 *DAG.getContext(),
2799 Guard.getValueType()),
2800 Guard, GuardVal, ISD::SETNE);
2801
2802 // If the guard/stackslot do not equal, branch to failure MBB.
2803 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2804 MVT::Other, GuardVal.getOperand(0),
2805 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2806 // Otherwise branch to success MBB.
2807 SDValue Br = DAG.getNode(ISD::BR, dl,
2808 MVT::Other, BrCond,
2810
2811 DAG.setRoot(Br);
2812}
2813
2814/// Codegen the failure basic block for a stack protector check.
2815///
2816/// A failure stack protector machine basic block consists simply of a call to
2817/// __stack_chk_fail().
2818///
2819/// For a high level explanation of how this fits into the stack protector
2820/// generation see the comment on the declaration of class
2821/// StackProtectorDescriptor.
2822void
2826 CallOptions.setDiscardResult(true);
2827 SDValue Chain =
2828 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2829 std::nullopt, CallOptions, getCurSDLoc())
2830 .second;
2831 // On PS4/PS5, the "return address" must still be within the calling
2832 // function, even if it's at the very end, so emit an explicit TRAP here.
2833 // Passing 'true' for doesNotReturn above won't generate the trap for us.
2834 if (TM.getTargetTriple().isPS())
2835 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2836 // WebAssembly needs an unreachable instruction after a non-returning call,
2837 // because the function return type can be different from __stack_chk_fail's
2838 // return type (void).
2839 if (TM.getTargetTriple().isWasm())
2840 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2841
2842 DAG.setRoot(Chain);
2843}
2844
2845/// visitBitTestHeader - This function emits necessary code to produce value
2846/// suitable for "bit tests"
2848 MachineBasicBlock *SwitchBB) {
2849 SDLoc dl = getCurSDLoc();
2850
2851 // Subtract the minimum value.
2852 SDValue SwitchOp = getValue(B.SValue);
2853 EVT VT = SwitchOp.getValueType();
2854 SDValue RangeSub =
2855 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
2856
2857 // Determine the type of the test operands.
2859 bool UsePtrType = false;
2860 if (!TLI.isTypeLegal(VT)) {
2861 UsePtrType = true;
2862 } else {
2863 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2864 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2865 // Switch table case range are encoded into series of masks.
2866 // Just use pointer type, it's guaranteed to fit.
2867 UsePtrType = true;
2868 break;
2869 }
2870 }
2871 SDValue Sub = RangeSub;
2872 if (UsePtrType) {
2873 VT = TLI.getPointerTy(DAG.getDataLayout());
2874 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2875 }
2876
2877 B.RegVT = VT.getSimpleVT();
2878 B.Reg = FuncInfo.CreateReg(B.RegVT);
2879 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2880
2881 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2882
2883 if (!B.FallthroughUnreachable)
2884 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2885 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2886 SwitchBB->normalizeSuccProbs();
2887
2888 SDValue Root = CopyTo;
2889 if (!B.FallthroughUnreachable) {
2890 // Conditional branch to the default block.
2891 SDValue RangeCmp = DAG.getSetCC(dl,
2893 RangeSub.getValueType()),
2894 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
2895 ISD::SETUGT);
2896
2897 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
2898 DAG.getBasicBlock(B.Default));
2899 }
2900
2901 // Avoid emitting unnecessary branches to the next block.
2902 if (MBB != NextBlock(SwitchBB))
2903 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
2904
2905 DAG.setRoot(Root);
2906}
2907
2908/// visitBitTestCase - this function produces one "bit test"
2910 MachineBasicBlock* NextMBB,
2911 BranchProbability BranchProbToNext,
2912 unsigned Reg,
2913 BitTestCase &B,
2914 MachineBasicBlock *SwitchBB) {
2915 SDLoc dl = getCurSDLoc();
2916 MVT VT = BB.RegVT;
2917 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2918 SDValue Cmp;
2919 unsigned PopCount = llvm::popcount(B.Mask);
2921 if (PopCount == 1) {
2922 // Testing for a single bit; just compare the shift count with what it
2923 // would need to be to shift a 1 bit in that position.
2924 Cmp = DAG.getSetCC(
2926 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
2927 ISD::SETEQ);
2928 } else if (PopCount == BB.Range) {
2929 // There is only one zero bit in the range, test for it directly.
2930 Cmp = DAG.getSetCC(
2932 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
2933 } else {
2934 // Make desired shift
2935 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2936 DAG.getConstant(1, dl, VT), ShiftOp);
2937
2938 // Emit bit tests and jumps
2939 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2940 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2941 Cmp = DAG.getSetCC(
2943 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2944 }
2945
2946 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2947 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2948 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2949 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2950 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2951 // one as they are relative probabilities (and thus work more like weights),
2952 // and hence we need to normalize them to let the sum of them become one.
2953 SwitchBB->normalizeSuccProbs();
2954
2955 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2957 Cmp, DAG.getBasicBlock(B.TargetBB));
2958
2959 // Avoid emitting unnecessary branches to the next block.
2960 if (NextMBB != NextBlock(SwitchBB))
2961 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2962 DAG.getBasicBlock(NextMBB));
2963
2964 DAG.setRoot(BrAnd);
2965}
2966
2967void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2968 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2969
2970 // Retrieve successors. Look through artificial IR level blocks like
2971 // catchswitch for successors.
2972 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2973 const BasicBlock *EHPadBB = I.getSuccessor(1);
2974
2975 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2976 // have to do anything here to lower funclet bundles.
2977 assert(!I.hasOperandBundlesOtherThan(
2978 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
2979 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
2980 LLVMContext::OB_cfguardtarget,
2981 LLVMContext::OB_clang_arc_attachedcall}) &&
2982 "Cannot lower invokes with arbitrary operand bundles yet!");
2983
2984 const Value *Callee(I.getCalledOperand());
2985 const Function *Fn = dyn_cast<Function>(Callee);
2986 if (isa<InlineAsm>(Callee))
2987 visitInlineAsm(I, EHPadBB);
2988 else if (Fn && Fn->isIntrinsic()) {
2989 switch (Fn->getIntrinsicID()) {
2990 default:
2991 llvm_unreachable("Cannot invoke this intrinsic");
2992 case Intrinsic::donothing:
2993 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2994 case Intrinsic::seh_try_begin:
2995 case Intrinsic::seh_scope_begin:
2996 case Intrinsic::seh_try_end:
2997 case Intrinsic::seh_scope_end:
2998 break;
2999 case Intrinsic::experimental_patchpoint_void:
3000 case Intrinsic::experimental_patchpoint_i64:
3001 visitPatchpoint(I, EHPadBB);
3002 break;
3003 case Intrinsic::experimental_gc_statepoint:
3004 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3005 break;
3006 case Intrinsic::wasm_rethrow: {
3007 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3008 // special because it can be invoked, so we manually lower it to a DAG
3009 // node here.
3011 Ops.push_back(getRoot()); // inchain
3013 Ops.push_back(
3014 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3016 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3018 break;
3019 }
3020 }
3021 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
3022 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3023 // Eventually we will support lowering the @llvm.experimental.deoptimize
3024 // intrinsic, and right now there are no plans to support other intrinsics
3025 // with deopt state.
3027 } else {
3028 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3029 }
3030
3031 // If the value of the invoke is used outside of its defining block, make it
3032 // available as a virtual register.
3033 // We already took care of the exported value for the statepoint instruction
3034 // during call to the LowerStatepoint.
3035 if (!isa<GCStatepointInst>(I)) {
3037 }
3038
3041 BranchProbability EHPadBBProb =
3042 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3044 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3045
3046 // Update successor info.
3047 addSuccessorWithProb(InvokeMBB, Return);
3048 for (auto &UnwindDest : UnwindDests) {
3049 UnwindDest.first->setIsEHPad();
3050 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3051 }
3052 InvokeMBB->normalizeSuccProbs();
3053
3054 // Drop into normal successor.
3056 DAG.getBasicBlock(Return)));
3057}
3058
3059void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3060 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3061
3062 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3063 // have to do anything here to lower funclet bundles.
3064 assert(!I.hasOperandBundlesOtherThan(
3065 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3066 "Cannot lower callbrs with arbitrary operand bundles yet!");
3067
3068 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3069 visitInlineAsm(I);
3071
3072 // Retrieve successors.
3074 Dests.insert(I.getDefaultDest());
3075 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
3076
3077 // Update successor info.
3078 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3079 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3080 BasicBlock *Dest = I.getIndirectDest(i);
3081 MachineBasicBlock *Target = FuncInfo.MBBMap[Dest];
3082 Target->setIsInlineAsmBrIndirectTarget();
3083 Target->setMachineBlockAddressTaken();
3084 Target->setLabelMustBeEmitted();
3085 // Don't add duplicate machine successors.
3086 if (Dests.insert(Dest).second)
3087 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3088 }
3089 CallBrMBB->normalizeSuccProbs();
3090
3091 // Drop into default successor.
3094 DAG.getBasicBlock(Return)));
3095}
3096
3097void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3098 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3099}
3100
3101void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3102 assert(FuncInfo.MBB->isEHPad() &&
3103 "Call to landingpad not in landing pad!");
3104
3105 // If there aren't registers to copy the values into (e.g., during SjLj
3106 // exceptions), then don't bother to create these DAG nodes.
3108 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3109 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3110 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3111 return;
3112
3113 // If landingpad's return type is token type, we don't create DAG nodes
3114 // for its exception pointer and selector value. The extraction of exception
3115 // pointer or selector value from token type landingpads is not currently
3116 // supported.
3117 if (LP.getType()->isTokenTy())
3118 return;
3119
3120 SmallVector<EVT, 2> ValueVTs;
3121 SDLoc dl = getCurSDLoc();
3122 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3123 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3124
3125 // Get the two live-in registers as SDValues. The physregs have already been
3126 // copied into virtual registers.
3127 SDValue Ops[2];
3128 if (FuncInfo.ExceptionPointerVirtReg) {
3129 Ops[0] = DAG.getZExtOrTrunc(
3131 FuncInfo.ExceptionPointerVirtReg,
3133 dl, ValueVTs[0]);
3134 } else {
3135 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3136 }
3137 Ops[1] = DAG.getZExtOrTrunc(
3139 FuncInfo.ExceptionSelectorVirtReg,
3141 dl, ValueVTs[1]);
3142
3143 // Merge into one.
3145 DAG.getVTList(ValueVTs), Ops);
3146 setValue(&LP, Res);
3147}
3148
3151 // Update JTCases.
3152 for (JumpTableBlock &JTB : SL->JTCases)
3153 if (JTB.first.HeaderBB == First)
3154 JTB.first.HeaderBB = Last;
3155
3156 // Update BitTestCases.
3157 for (BitTestBlock &BTB : SL->BitTestCases)
3158 if (BTB.Parent == First)
3159 BTB.Parent = Last;
3160}
3161
3162void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3163 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3164
3165 // Update machine-CFG edges with unique successors.
3167 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3168 BasicBlock *BB = I.getSuccessor(i);
3169 bool Inserted = Done.insert(BB).second;
3170 if (!Inserted)
3171 continue;
3172
3173 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
3174 addSuccessorWithProb(IndirectBrMBB, Succ);
3175 }
3176 IndirectBrMBB->normalizeSuccProbs();
3177
3180 getValue(I.getAddress())));
3181}
3182
3183void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3185 return;
3186
3187 // We may be able to ignore unreachable behind a noreturn call.
3189 const BasicBlock &BB = *I.getParent();
3190 if (&I != &BB.front()) {
3192 std::prev(BasicBlock::const_iterator(&I));
3193 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
3194 if (Call->doesNotReturn())
3195 return;
3196 }
3197 }
3198 }
3199
3201}
3202
3203void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3205 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3206 Flags.copyFMF(*FPOp);
3207
3208 SDValue Op = getValue(I.getOperand(0));
3209 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3210 Op, Flags);
3211 setValue(&I, UnNodeValue);
3212}
3213
3214void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3216 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3217 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3218 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3219 }
3220 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3221 Flags.setExact(ExactOp->isExact());
3222 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3223 Flags.copyFMF(*FPOp);
3224
3225 SDValue Op1 = getValue(I.getOperand(0));
3226 SDValue Op2 = getValue(I.getOperand(1));
3227 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3228 Op1, Op2, Flags);
3229 setValue(&I, BinNodeValue);
3230}
3231
3232void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3233 SDValue Op1 = getValue(I.getOperand(0));
3234 SDValue Op2 = getValue(I.getOperand(1));
3235
3237 Op1.getValueType(), DAG.getDataLayout());
3238
3239 // Coerce the shift amount to the right type if we can. This exposes the
3240 // truncate or zext to optimization early.
3241 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3243 "Unexpected shift type");
3244 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3245 }
3246
3247 bool nuw = false;
3248 bool nsw = false;
3249 bool exact = false;
3250
3251 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3252
3253 if (const OverflowingBinaryOperator *OFBinOp =
3254 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3255 nuw = OFBinOp->hasNoUnsignedWrap();
3256 nsw = OFBinOp->hasNoSignedWrap();
3257 }
3258 if (const PossiblyExactOperator *ExactOp =
3259 dyn_cast<const PossiblyExactOperator>(&I))
3260 exact = ExactOp->isExact();
3261 }
3263 Flags.setExact(exact);
3264 Flags.setNoSignedWrap(nsw);
3265 Flags.setNoUnsignedWrap(nuw);
3266 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3267 Flags);
3268 setValue(&I, Res);
3269}
3270
3271void SelectionDAGBuilder::visitSDiv(const User &I) {
3272 SDValue Op1 = getValue(I.getOperand(0));
3273 SDValue Op2 = getValue(I.getOperand(1));
3274
3276 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3277 cast<PossiblyExactOperator>(&I)->isExact());
3279 Op2, Flags));
3280}
3281
3282void SelectionDAGBuilder::visitICmp(const User &I) {
3284 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3285 predicate = IC->getPredicate();
3286 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3287 predicate = ICmpInst::Predicate(IC->getPredicate());
3288 SDValue Op1 = getValue(I.getOperand(0));
3289 SDValue Op2 = getValue(I.getOperand(1));
3290 ISD::CondCode Opcode = getICmpCondCode(predicate);
3291
3292 auto &TLI = DAG.getTargetLoweringInfo();
3293 EVT MemVT =
3294 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3295
3296 // If a pointer's DAG type is larger than its memory type then the DAG values
3297 // are zero-extended. This breaks signed comparisons so truncate back to the
3298 // underlying type before doing the compare.
3299 if (Op1.getValueType() != MemVT) {
3300 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3301 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3302 }
3303
3305 I.getType());
3306 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3307}
3308
3309void SelectionDAGBuilder::visitFCmp(const User &I) {
3311 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3312 predicate = FC->getPredicate();
3313 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3314 predicate = FCmpInst::Predicate(FC->getPredicate());
3315 SDValue Op1 = getValue(I.getOperand(0));
3316 SDValue Op2 = getValue(I.getOperand(1));
3317
3318 ISD::CondCode Condition = getFCmpCondCode(predicate);
3319 auto *FPMO = cast<FPMathOperator>(&I);
3320 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3321 Condition = getFCmpCodeWithoutNaN(Condition);
3322
3324 Flags.copyFMF(*FPMO);
3325 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3326
3328 I.getType());
3329 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3330}
3331
3332// Check if the condition of the select has one use or two users that are both
3333// selects with the same condition.
3334static bool hasOnlySelectUsers(const Value *Cond) {
3335 return llvm::all_of(Cond->users(), [](const Value *V) {
3336 return isa<SelectInst>(V);
3337 });
3338}
3339
3340void SelectionDAGBuilder::visitSelect(const User &I) {
3341 SmallVector<EVT, 4> ValueVTs;
3343 ValueVTs);
3344 unsigned NumValues = ValueVTs.size();
3345 if (NumValues == 0) return;
3346
3347 SmallVector<SDValue, 4> Values(NumValues);
3348 SDValue Cond = getValue(I.getOperand(0));
3349 SDValue LHSVal = getValue(I.getOperand(1));
3350 SDValue RHSVal = getValue(I.getOperand(2));
3351 SmallVector<SDValue, 1> BaseOps(1, Cond);
3352 ISD::NodeType OpCode =
3353 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3354
3355 bool IsUnaryAbs = false;
3356 bool Negate = false;
3357
3359 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3360 Flags.copyFMF(*FPOp);
3361
3362 // Min/max matching is only viable if all output VTs are the same.
3363 if (all_equal(ValueVTs)) {
3364 EVT VT = ValueVTs[0];
3365 LLVMContext &Ctx = *DAG.getContext();
3366 auto &TLI = DAG.getTargetLoweringInfo();
3367
3368 // We care about the legality of the operation after it has been type
3369 // legalized.
3370 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3371 VT = TLI.getTypeToTransformTo(Ctx, VT);
3372
3373 // If the vselect is legal, assume we want to leave this as a vector setcc +
3374 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3375 // min/max is legal on the scalar type.
3376 bool UseScalarMinMax = VT.isVector() &&
3378
3379 // ValueTracking's select pattern matching does not account for -0.0,
3380 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3381 // -0.0 is less than +0.0.
3382 Value *LHS, *RHS;
3383 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3385 switch (SPR.Flavor) {
3386 case SPF_UMAX: Opc = ISD::UMAX; break;
3387 case SPF_UMIN: Opc = ISD::UMIN; break;
3388 case SPF_SMAX: Opc = ISD::SMAX; break;
3389 case SPF_SMIN: Opc = ISD::SMIN; break;
3390 case SPF_FMINNUM:
3391 switch (SPR.NaNBehavior) {
3392 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3393 case SPNB_RETURNS_NAN: break;
3394 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3395 case SPNB_RETURNS_ANY:
3397 (UseScalarMinMax &&
3399 Opc = ISD::FMINNUM;
3400 break;
3401 }
3402 break;
3403 case SPF_FMAXNUM:
3404 switch (SPR.NaNBehavior) {
3405 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3406 case SPNB_RETURNS_NAN: break;
3407 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3408 case SPNB_RETURNS_ANY:
3410 (UseScalarMinMax &&
3412 Opc = ISD::FMAXNUM;
3413 break;
3414 }
3415 break;
3416 case SPF_NABS:
3417 Negate = true;
3418 [[fallthrough]];
3419 case SPF_ABS:
3420 IsUnaryAbs = true;
3421 Opc = ISD::ABS;
3422 break;
3423 default: break;
3424 }
3425
3426 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3427 (TLI.isOperationLegalOrCustom(Opc, VT) ||
3428 (UseScalarMinMax &&
3429 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3430 // If the underlying comparison instruction is used by any other
3431 // instruction, the consumed instructions won't be destroyed, so it is
3432 // not profitable to convert to a min/max.
3433 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3434 OpCode = Opc;
3435 LHSVal = getValue(LHS);
3436 RHSVal = getValue(RHS);
3437 BaseOps.clear();
3438 }
3439
3440 if (IsUnaryAbs) {
3441 OpCode = Opc;
3442 LHSVal = getValue(LHS);
3443 BaseOps.clear();
3444 }
3445 }
3446
3447 if (IsUnaryAbs) {
3448 for (unsigned i = 0; i != NumValues; ++i) {
3449 SDLoc dl = getCurSDLoc();
3450 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3451 Values[i] =
3452 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3453 if (Negate)
3454 Values[i] = DAG.getNegative(Values[i], dl, VT);
3455 }
3456 } else {
3457 for (unsigned i = 0; i != NumValues; ++i) {
3458 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3459 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3460 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3461 Values[i] = DAG.getNode(
3462 OpCode, getCurSDLoc(),
3463 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3464 }
3465 }
3466
3468 DAG.getVTList(ValueVTs), Values));
3469}
3470
3471void SelectionDAGBuilder::visitTrunc(const User &I) {
3472 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3473 SDValue N = getValue(I.getOperand(0));
3475 I.getType());
3477}
3478
3479void SelectionDAGBuilder::visitZExt(const User &I) {
3480 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3481 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3482 SDValue N = getValue(I.getOperand(0));
3484 I.getType());
3486}
3487
3488void SelectionDAGBuilder::visitSExt(const User &I) {
3489 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3490 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3491 SDValue N = getValue(I.getOperand(0));
3493 I.getType());
3495}
3496
3497void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3498 // FPTrunc is never a no-op cast, no need to check
3499 SDValue N = getValue(I.getOperand(0));
3500 SDLoc dl = getCurSDLoc();
3502 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3503 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3505 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3506}
3507
3508void SelectionDAGBuilder::visitFPExt(const User &I) {
3509 // FPExt is never a no-op cast, no need to check
3510 SDValue N = getValue(I.getOperand(0));
3512 I.getType());
3514}
3515
3516void SelectionDAGBuilder::visitFPToUI(const User &I) {
3517 // FPToUI is never a no-op cast, no need to check
3518 SDValue N = getValue(I.getOperand(0));
3520 I.getType());
3522}
3523
3524void SelectionDAGBuilder::visitFPToSI(const User &I) {
3525 // FPToSI is never a no-op cast, no need to check
3526 SDValue N = getValue(I.getOperand(0));
3528 I.getType());
3530}
3531
3532void SelectionDAGBuilder::visitUIToFP(const User &I) {
3533 // UIToFP is never a no-op cast, no need to check
3534 SDValue N = getValue(I.getOperand(0));
3536 I.getType());
3538}
3539
3540void SelectionDAGBuilder::visitSIToFP(const User &I) {
3541 // SIToFP is never a no-op cast, no need to check
3542 SDValue N = getValue(I.getOperand(0));
3544 I.getType());
3546}
3547
3548void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3549 // What to do depends on the size of the integer and the size of the pointer.
3550 // We can either truncate, zero extend, or no-op, accordingly.
3551 SDValue N = getValue(I.getOperand(0));
3552 auto &TLI = DAG.getTargetLoweringInfo();
3554 I.getType());
3555 EVT PtrMemVT =
3556 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3557 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3558 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3559 setValue(&I, N);
3560}
3561
3562void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3563 // What to do depends on the size of the integer and the size of the pointer.
3564 // We can either truncate, zero extend, or no-op, accordingly.
3565 SDValue N = getValue(I.getOperand(0));
3566 auto &TLI = DAG.getTargetLoweringInfo();
3567 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3568 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3569 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3570 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3571 setValue(&I, N);
3572}
3573
3574void SelectionDAGBuilder::visitBitCast(const User &I) {
3575 SDValue N = getValue(I.getOperand(0));
3576 SDLoc dl = getCurSDLoc();
3578 I.getType());
3579
3580 // BitCast assures us that source and destination are the same size so this is
3581 // either a BITCAST or a no-op.
3582 if (DestVT != N.getValueType())
3584 DestVT, N)); // convert types.
3585 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3586 // might fold any kind of constant expression to an integer constant and that
3587 // is not what we are looking for. Only recognize a bitcast of a genuine
3588 // constant integer as an opaque constant.
3589 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3590 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3591 /*isOpaque*/true));
3592 else
3593 setValue(&I, N); // noop cast.
3594}
3595
3596void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3598 const Value *SV = I.getOperand(0);
3599 SDValue N = getValue(SV);
3600 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3601
3602 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3603 unsigned DestAS = I.getType()->getPointerAddressSpace();
3604
3605 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3606 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3607
3608 setValue(&I, N);
3609}
3610
3611void SelectionDAGBuilder::visitInsertElement(const User &I) {
3613 SDValue InVec = getValue(I.getOperand(0));
3614 SDValue InVal = getValue(I.getOperand(1));
3615 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3618 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3619 InVec, InVal, InIdx));
3620}
3621
3622void SelectionDAGBuilder::visitExtractElement(const User &I) {
3624 SDValue InVec = getValue(I.getOperand(0));
3625 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3628 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3629 InVec, InIdx));
3630}
3631
3632void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3633 SDValue Src1 = getValue(I.getOperand(0));
3634 SDValue Src2 = getValue(I.getOperand(1));
3636 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
3637 Mask = SVI->getShuffleMask();
3638 else
3639 Mask = cast<ConstantExpr>(I).getShuffleMask();
3640 SDLoc DL = getCurSDLoc();
3642 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3643 EVT SrcVT = Src1.getValueType();
3644
3645 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
3646 VT.isScalableVector()) {
3647 // Canonical splat form of first element of first input vector.
3648 SDValue FirstElt =
3651 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
3652 return;
3653 }
3654
3655 // For now, we only handle splats for scalable vectors.
3656 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
3657 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
3658 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
3659
3660 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3661 unsigned MaskNumElts = Mask.size();
3662
3663 if (SrcNumElts == MaskNumElts) {
3664 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3665 return;
3666 }
3667
3668 // Normalize the shuffle vector since mask and vector length don't match.
3669 if (SrcNumElts < MaskNumElts) {
3670 // Mask is longer than the source vectors. We can use concatenate vector to
3671 // make the mask and vectors lengths match.
3672
3673 if (MaskNumElts % SrcNumElts == 0) {
3674 // Mask length is a multiple of the source vector length.
3675 // Check if the shuffle is some kind of concatenation of the input
3676 // vectors.
3677 unsigned NumConcat = MaskNumElts / SrcNumElts;
3678 bool IsConcat = true;
3679 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3680 for (unsigned i = 0; i != MaskNumElts; ++i) {
3681 int Idx = Mask[i];
3682 if (Idx < 0)
3683 continue;
3684 // Ensure the indices in each SrcVT sized piece are sequential and that
3685 // the same source is used for the whole piece.
3686 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3687 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3688 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3689 IsConcat = false;
3690 break;
3691 }
3692 // Remember which source this index came from.
3693 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3694 }
3695
3696 // The shuffle is concatenating multiple vectors together. Just emit
3697 // a CONCAT_VECTORS operation.
3698 if (IsConcat) {
3699 SmallVector<SDValue, 8> ConcatOps;
3700 for (auto Src : ConcatSrcs) {
3701 if (Src < 0)
3702 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3703 else if (Src == 0)
3704 ConcatOps.push_back(Src1);
3705 else
3706 ConcatOps.push_back(Src2);
3707 }
3708 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3709 return;
3710 }
3711 }
3712
3713 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3714 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3715 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3716 PaddedMaskNumElts);
3717
3718 // Pad both vectors with undefs to make them the same length as the mask.
3719 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3720
3721 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3722 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3723 MOps1[0] = Src1;
3724 MOps2[0] = Src2;
3725
3726 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3727 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3728
3729 // Readjust mask for new input vector length.
3730 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3731 for (unsigned i = 0; i != MaskNumElts; ++i) {
3732 int Idx = Mask[i];
3733 if (Idx >= (int)SrcNumElts)
3734 Idx -= SrcNumElts - PaddedMaskNumElts;
3735 MappedOps[i] = Idx;
3736 }
3737
3738 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3739
3740 // If the concatenated vector was padded, extract a subvector with the
3741 // correct number of elements.
3742 if (MaskNumElts != PaddedMaskNumElts)
3745
3746 setValue(&I, Result);
3747 return;
3748 }
3749
3750 if (SrcNumElts > MaskNumElts) {
3751 // Analyze the access pattern of the vector to see if we can extract
3752 // two subvectors and do the shuffle.
3753 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3754 bool CanExtract = true;
3755 for (int Idx : Mask) {
3756 unsigned Input = 0;
3757 if (Idx < 0)
3758 continue;
3759
3760 if (Idx >= (int)SrcNumElts) {
3761 Input = 1;
3762 Idx -= SrcNumElts;
3763 }
3764
3765 // If all the indices come from the same MaskNumElts sized portion of
3766 // the sources we can use extract. Also make sure the extract wouldn't
3767 // extract past the end of the source.
3768 int NewStartIdx = alignDown(Idx, MaskNumElts);
3769 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3770 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3771 CanExtract = false;
3772 // Make sure we always update StartIdx as we use it to track if all
3773 // elements are undef.
3774 StartIdx[Input] = NewStartIdx;
3775 }
3776
3777 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3778 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3779 return;
3780 }
3781 if (CanExtract) {
3782 // Extract appropriate subvector and generate a vector shuffle
3783 for (unsigned Input = 0; Input < 2; ++Input) {
3784 SDValue &Src = Input == 0 ? Src1 : Src2;
3785 if (StartIdx[Input] < 0)
3786 Src = DAG.getUNDEF(VT);
3787 else {
3788 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3789 DAG.getVectorIdxConstant(StartIdx[Input], DL));
3790 }
3791 }
3792
3793 // Calculate new mask.
3794 SmallVector<int, 8> MappedOps(Mask);
3795 for (int &Idx : MappedOps) {
3796 if (Idx >= (int)SrcNumElts)
3797 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3798 else if (Idx >= 0)
3799 Idx -= StartIdx[0];
3800 }
3801
3802 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3803 return;
3804 }
3805 }
3806
3807 // We can't use either concat vectors or extract subvectors so fall back to
3808 // replacing the shuffle with extract and build vector.
3809 // to insert and build vector.
3810 EVT EltVT = VT.getVectorElementType();
3812 for (int Idx : Mask) {
3813 SDValue Res;
3814
3815 if (Idx < 0) {
3816 Res = DAG.getUNDEF(EltVT);
3817 } else {
3818 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3819 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3820
3821 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
3823 }
3824
3825 Ops.push_back(Res);
3826 }
3827
3828 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3829}
3830
3831void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
3832 ArrayRef<unsigned> Indices = I.getIndices();
3833 const Value *Op0 = I.getOperand(0);
3834 const Value *Op1 = I.getOperand(1);
3835 Type *AggTy = I.getType();
3836 Type *ValTy = Op1->getType();
3837 bool IntoUndef = isa<UndefValue>(Op0);
3838 bool FromUndef = isa<UndefValue>(Op1);
3839
3840 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3841
3843 SmallVector<EVT, 4> AggValueVTs;
3844 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3845 SmallVector<EVT, 4> ValValueVTs;
3846 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3847
3848 unsigned NumAggValues = AggValueVTs.size();
3849 unsigned NumValValues = ValValueVTs.size();
3850 SmallVector<SDValue, 4> Values(NumAggValues);
3851
3852 // Ignore an insertvalue that produces an empty object
3853 if (!NumAggValues) {
3855 return;
3856 }
3857
3858 SDValue Agg = getValue(Op0);
3859 unsigned i = 0;
3860 // Copy the beginning value(s) from the original aggregate.
3861 for (; i != LinearIndex; ++i)
3862 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3863 SDValue(Agg.getNode(), Agg.getResNo() + i);
3864 // Copy values from the inserted value(s).
3865 if (NumValValues) {
3866 SDValue Val = getValue(Op1);
3867 for (; i != LinearIndex + NumValValues; ++i)
3868 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3869 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3870 }
3871 // Copy remaining value(s) from the original aggregate.
3872 for (; i != NumAggValues; ++i)
3873 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3874 SDValue(Agg.getNode(), Agg.getResNo() + i);
3875
3877 DAG.getVTList(AggValueVTs), Values));
3878}
3879
3880void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
3881 ArrayRef<unsigned> Indices = I.getIndices();
3882 const Value *Op0 = I.getOperand(0);
3883 Type *AggTy = Op0->getType();
3884 Type *ValTy = I.getType();
3885 bool OutOfUndef = isa<UndefValue>(Op0);
3886
3887 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3888
3890 SmallVector<EVT, 4> ValValueVTs;
3891 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3892
3893 unsigned NumValValues = ValValueVTs.size();
3894
3895 // Ignore a extractvalue that produces an empty object
3896 if (!NumValValues) {
3898 return;
3899 }
3900
3901 SmallVector<SDValue, 4> Values(NumValValues);
3902
3903 SDValue Agg = getValue(Op0);
3904 // Copy out the selected value(s).
3905 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3906 Values[i - LinearIndex] =
3907 OutOfUndef ?
3908 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3909 SDValue(Agg.getNode(), Agg.getResNo() + i);
3910
3912 DAG.getVTList(ValValueVTs), Values));
3913}
3914
3915void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3916 Value *Op0 = I.getOperand(0);
3917 // Note that the pointer operand may be a vector of pointers. Take the scalar
3918 // element which holds a pointer.
3919 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3920 SDValue N = getValue(Op0);
3921 SDLoc dl = getCurSDLoc();
3922 auto &TLI = DAG.getTargetLoweringInfo();
3923
3924 // Normalize Vector GEP - all scalar operands should be converted to the
3925 // splat vector.
3926 bool IsVectorGEP = I.getType()->isVectorTy();
3927 ElementCount VectorElementCount =
3928 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
3930
3931 if (IsVectorGEP && !N.getValueType().isVector()) {
3933 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
3934 N = DAG.getSplat(VT, dl, N);
3935 }
3936
3938 GTI != E; ++GTI) {
3939 const Value *Idx = GTI.getOperand();
3940 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3941 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3942 if (Field) {
3943 // N = N + Offset
3946
3947 // In an inbounds GEP with an offset that is nonnegative even when
3948 // interpreted as signed, assume there is no unsigned overflow.
3950 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3951 Flags.setNoUnsignedWrap(true);
3952
3953 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3954 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3955 }
3956 } else {
3957 // IdxSize is the width of the arithmetic according to IR semantics.
3958 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
3959 // (and fix up the result later).
3960 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3961 MVT IdxTy = MVT::getIntegerVT(IdxSize);
3962 TypeSize ElementSize =
3963 DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
3964 // We intentionally mask away the high bits here; ElementSize may not
3965 // fit in IdxTy.
3966 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
3967 bool ElementScalable = ElementSize.isScalable();
3968
3969 // If this is a scalar constant or a splat vector of constants,
3970 // handle it quickly.
3971 const auto *C = dyn_cast<Constant>(Idx);
3972 if (C && isa<VectorType>(C->getType()))
3973 C = C->getSplatValue();
3974
3975 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
3976 if (CI && CI->isZero())
3977 continue;
3978 if (CI && !ElementScalable) {
3979 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
3981 SDValue OffsVal;
3982 if (IsVectorGEP)
3983 OffsVal = DAG.getConstant(
3984 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
3985 else
3986 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
3987
3988 // In an inbounds GEP with an offset that is nonnegative even when
3989 // interpreted as signed, assume there is no unsigned overflow.
3991 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3992 Flags.setNoUnsignedWrap(true);
3993
3994 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3995
3996 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3997 continue;
3998 }
3999
4000 // N = N + Idx * ElementMul;
4001 SDValue IdxN = getValue(Idx);
4002
4003 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4005 VectorElementCount);
4006 IdxN = DAG.getSplat(VT, dl, IdxN);
4007 }
4008
4009 // If the index is smaller or larger than intptr_t, truncate or extend
4010 // it.
4011 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4012
4013 if (ElementScalable) {
4014 EVT VScaleTy = N.getValueType().getScalarType();
4015 SDValue VScale = DAG.getNode(
4016 ISD::VSCALE, dl, VScaleTy,
4017 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4018 if (IsVectorGEP)
4019 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4020 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
4021 } else {
4022 // If this is a multiply by a power of two, turn it into a shl
4023 // immediately. This is a very common case.
4024 if (ElementMul != 1) {
4025 if (ElementMul.isPowerOf2()) {
4026 unsigned Amt = ElementMul.logBase2();
4027 IdxN = DAG.getNode(ISD::SHL, dl,
4028 N.getValueType(), IdxN,
4029 DAG.getConstant(Amt, dl, IdxN.getValueType()));
4030 } else {
4031 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4032 IdxN.getValueType());
4033 IdxN = DAG.getNode(ISD::MUL, dl,
4034 N.getValueType(), IdxN, Scale);
4035 }
4036 }
4037 }
4038
4039 N = DAG.getNode(ISD::ADD, dl,
4040 N.getValueType(), N, IdxN);
4041 }
4042 }
4043
4044 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4045 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4046 if (IsVectorGEP) {
4047 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4048 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4049 }
4050
4051 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4052 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4053
4054 setValue(&I, N);
4055}
4056
4057void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4058 // If this is a fixed sized alloca in the entry block of the function,
4059 // allocate it statically on the stack.
4060 if (FuncInfo.StaticAllocaMap.count(&I))
4061 return; // getValue will auto-populate this.
4062
4063 SDLoc dl = getCurSDLoc();
4064 Type *Ty = I.getAllocatedType();
4066 auto &DL = DAG.getDataLayout();
4067 TypeSize TySize = DL.getTypeAllocSize(Ty);
4068 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4069
4070 SDValue AllocSize = getValue(I.getArraySize());
4071
4072 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace());
4073 if (AllocSize.getValueType() != IntPtr)
4074 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4075
4076 if (TySize.isScalable())
4077 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4078 DAG.getVScale(dl, IntPtr,
4079 APInt(IntPtr.getScalarSizeInBits(),
4080 TySize.getKnownMinValue())));
4081 else
4082 AllocSize =
4083 DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4084 DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));
4085
4086 // Handle alignment. If the requested alignment is less than or equal to
4087 // the stack alignment, ignore it. If the size is greater than or equal to
4088 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4090 if (*Alignment <= StackAlign)
4091 Alignment = std::nullopt;
4092
4093 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4094 // Round the size of the allocation up to the stack alignment size
4095 // by add SA-1 to the size. This doesn't overflow because we're computing
4096 // an address inside an alloca.
4098 Flags.setNoUnsignedWrap(true);
4099 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4100 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);
4101
4102 // Mask out the low bits for alignment purposes.
4103 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4104 DAG.getConstant(~StackAlignMask, dl, IntPtr));
4105
4106 SDValue Ops[] = {
4107 getRoot(), AllocSize,
4108 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4109 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4110 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4111 setValue(&I, DSA);
4112 DAG.setRoot(DSA.getValue(1));
4113
4114 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
4115}
4116
4117void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4118 if (I.isAtomic())
4119 return visitAtomicLoad(I);
4120
4122 const Value *SV = I.getOperand(0);
4123 if (TLI.supportSwiftError()) {
4124 // Swifterror values can come from either a function parameter with
4125 // swifterror attribute or an alloca with swifterror attribute.
4126 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4127 if (Arg->hasSwiftErrorAttr())
4128 return visitLoadFromSwiftError(I);
4129 }
4130
4131 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4132 if (Alloca->isSwiftError())
4133 return visitLoadFromSwiftError(I);
4134 }
4135 }
4136
4137 SDValue Ptr = getValue(SV);
4138
4139 Type *Ty = I.getType();
4140 SmallVector<EVT, 4> ValueVTs, MemVTs;
4142 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4143 unsigned NumValues = ValueVTs.size();
4144 if (NumValues == 0)
4145 return;
4146
4147 Align Alignment = I.getAlign();
4148 AAMDNodes AAInfo = I.getAAMetadata();
4149 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4150 bool isVolatile = I.isVolatile();
4151 MachineMemOperand::Flags MMOFlags =
4153
4154 SDValue Root;
4155 bool ConstantMemory = false;
4156 if (isVolatile)
4157 // Serialize volatile loads with other side effects.
4158 Root = getRoot();
4159 else if (NumValues > MaxParallelChains)
4160 Root = getMemoryRoot();
4161 else if (AA &&
4163 SV,
4165 AAInfo))) {
4166 // Do not serialize (non-volatile) loads of constant memory with anything.
4167 Root = DAG.getEntryNode();
4168 ConstantMemory = true;
4170 } else {
4171 // Do not serialize non-volatile loads against each other.
4172 Root = DAG.getRoot();
4173 }
4174
4175 SDLoc dl = getCurSDLoc();
4176
4177 if (isVolatile)
4178 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4179 </