LLVM 19.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
19#include "llvm/ADT/SmallSet.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
45#include "llvm/Support/Debug.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 /// Some targets cannot handle a variable
122 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
123 /// is necessary to spill the vector being inserted into to memory, perform
124 /// the insert there, and then read the result back.
125 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
128 const SDLoc &dl);
129
130 /// Return a vector shuffle operation which
131 /// performs the same shuffe in terms of order or result bytes, but on a type
132 /// whose vector element type is narrower than the original shuffle type.
133 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
134 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
135 SDValue N1, SDValue N2,
136 ArrayRef<int> Mask) const;
137
138 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
140 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
141
142 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
145 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128,
150 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
151 RTLIB::Libcall Call_I8,
152 RTLIB::Libcall Call_I16,
153 RTLIB::Libcall Call_I32,
154 RTLIB::Libcall Call_I64,
155 RTLIB::Libcall Call_I128);
156 void ExpandArgFPLibCall(SDNode *Node,
157 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
158 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
159 RTLIB::Libcall Call_PPCF128,
161 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182
183 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
184 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
186 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
188 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
189
190 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
191
192 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
193 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
194 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
195
196 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
197 SDValue ExpandConstant(ConstantSDNode *CP);
198
199 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
200 bool ExpandNode(SDNode *Node);
201 void ConvertNodeToLibcall(SDNode *Node);
202 void PromoteNode(SDNode *Node);
203
204public:
205 // Node replacement helpers
206
207 void ReplacedNode(SDNode *N) {
208 LegalizedNodes.erase(N);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(N);
211 }
212
213 void ReplaceNode(SDNode *Old, SDNode *New) {
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
215 dbgs() << " with: "; New->dump(&DAG));
216
217 assert(Old->getNumValues() == New->getNumValues() &&
218 "Replacing one node with another that produces a different number "
219 "of values!");
220 DAG.ReplaceAllUsesWith(Old, New);
221 if (UpdatedNodes)
222 UpdatedNodes->insert(New);
223 ReplacedNode(Old);
224 }
225
226 void ReplaceNode(SDValue Old, SDValue New) {
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
228 dbgs() << " with: "; New->dump(&DAG));
229
230 DAG.ReplaceAllUsesWith(Old, New);
231 if (UpdatedNodes)
232 UpdatedNodes->insert(New.getNode());
233 ReplacedNode(Old.getNode());
234 }
235
236 void ReplaceNode(SDNode *Old, const SDValue *New) {
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(Old, New);
240 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
241 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
242 New[i]->dump(&DAG));
243 if (UpdatedNodes)
244 UpdatedNodes->insert(New[i].getNode());
245 }
246 ReplacedNode(Old);
247 }
248
249 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
251 dbgs() << " with: "; New->dump(&DAG));
252
253 DAG.ReplaceAllUsesOfValueWith(Old, New);
254 if (UpdatedNodes)
255 UpdatedNodes->insert(New.getNode());
256 ReplacedNode(Old.getNode());
257 }
258};
259
260} // end anonymous namespace
261
262// Helper function that generates an MMO that considers the alignment of the
263// stack, and the size of the stack object
265 MachineFunction &MF,
266 bool isObjectScalable) {
267 auto &MFI = MF.getFrameInfo();
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
270 uint64_t ObjectSize = isObjectScalable ? ~UINT64_C(0) : MFI.getObjectSize(FI);
272 ObjectSize, MFI.getObjectAlign(FI));
273}
274
275/// Return a vector shuffle operation which
276/// performs the same shuffle in terms of order or result bytes, but on a type
277/// whose vector element type is narrower than the original shuffle type.
278/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
279SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
280 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
281 ArrayRef<int> Mask) const {
282 unsigned NumMaskElts = VT.getVectorNumElements();
283 unsigned NumDestElts = NVT.getVectorNumElements();
284 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
285
286 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
287
288 if (NumEltsGrowth == 1)
289 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
290
291 SmallVector<int, 8> NewMask;
292 for (unsigned i = 0; i != NumMaskElts; ++i) {
293 int Idx = Mask[i];
294 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
295 if (Idx < 0)
296 NewMask.push_back(-1);
297 else
298 NewMask.push_back(Idx * NumEltsGrowth + j);
299 }
300 }
301 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
302 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
303 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
304}
305
306/// Expands the ConstantFP node to an integer constant or
307/// a load from the constant pool.
309SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
310 bool Extend = false;
311 SDLoc dl(CFP);
312
313 // If a FP immediate is precise when represented as a float and if the
314 // target can do an extending load from float to double, we put it into
315 // the constant pool as a float, even if it's is statically typed as a
316 // double. This shrinks FP constants and canonicalizes them for targets where
317 // an FP extending load is the same cost as a normal load (such as on the x87
318 // fp stack or PPC FP unit).
319 EVT VT = CFP->getValueType(0);
320 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
321 if (!UseCP) {
322 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
323 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
324 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
325 }
326
327 APFloat APF = CFP->getValueAPF();
328 EVT OrigVT = VT;
329 EVT SVT = VT;
330
331 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
332 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
333 if (!APF.isSignaling()) {
334 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
335 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
337 // Only do this if the target has a native EXTLOAD instruction from
338 // smaller type.
339 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
340 TLI.ShouldShrinkFPConstant(OrigVT)) {
341 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
342 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
343 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
344 VT = SVT;
345 Extend = true;
346 }
347 }
348 }
349
350 SDValue CPIdx =
351 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
352 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
353 if (Extend) {
354 SDValue Result = DAG.getExtLoad(
355 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
356 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
357 Alignment);
358 return Result;
359 }
360 SDValue Result = DAG.getLoad(
361 OrigVT, dl, DAG.getEntryNode(), CPIdx,
362 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
363 return Result;
364}
365
366/// Expands the Constant node to a load from the constant pool.
367SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
368 SDLoc dl(CP);
369 EVT VT = CP->getValueType(0);
370 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
371 TLI.getPointerTy(DAG.getDataLayout()));
372 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
373 SDValue Result = DAG.getLoad(
374 VT, dl, DAG.getEntryNode(), CPIdx,
375 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
376 return Result;
377}
378
379/// Some target cannot handle a variable insertion index for the
380/// INSERT_VECTOR_ELT instruction. In this case, it
381/// is necessary to spill the vector being inserted into to memory, perform
382/// the insert there, and then read the result back.
383SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
384 SDValue Val,
385 SDValue Idx,
386 const SDLoc &dl) {
387 SDValue Tmp1 = Vec;
388 SDValue Tmp2 = Val;
389 SDValue Tmp3 = Idx;
390
391 // If the target doesn't support this, we have to spill the input vector
392 // to a temporary stack slot, update the element, then reload it. This is
393 // badness. We could also load the value into a vector register (either
394 // with a "move to register" or "extload into register" instruction, then
395 // permute it into place, if the idx is a constant and if the idx is
396 // supported by the target.
397 EVT VT = Tmp1.getValueType();
398 EVT EltVT = VT.getVectorElementType();
399 SDValue StackPtr = DAG.CreateStackTemporary(VT);
400
401 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
402
403 // Store the vector.
404 SDValue Ch = DAG.getStore(
405 DAG.getEntryNode(), dl, Tmp1, StackPtr,
406 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
407
408 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
409
410 // Store the scalar value.
411 Ch = DAG.getTruncStore(
412 Ch, dl, Tmp2, StackPtr2,
413 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
414 // Load the updated vector.
415 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
416 DAG.getMachineFunction(), SPFI));
417}
418
419SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
420 SDValue Idx,
421 const SDLoc &dl) {
422 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
423 // SCALAR_TO_VECTOR requires that the type of the value being inserted
424 // match the element type of the vector being created, except for
425 // integers in which case the inserted value can be over width.
426 EVT EltVT = Vec.getValueType().getVectorElementType();
427 if (Val.getValueType() == EltVT ||
428 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
429 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
430 Vec.getValueType(), Val);
431
432 unsigned NumElts = Vec.getValueType().getVectorNumElements();
433 // We generate a shuffle of InVec and ScVec, so the shuffle mask
434 // should be 0,1,2,3,4,5... with the appropriate element replaced with
435 // elt 0 of the RHS.
436 SmallVector<int, 8> ShufOps;
437 for (unsigned i = 0; i != NumElts; ++i)
438 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
439
440 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
441 }
442 }
443 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
444}
445
446SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
447 if (!ISD::isNormalStore(ST))
448 return SDValue();
449
450 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
451 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
452 // FIXME: move this to the DAG Combiner! Note that we can't regress due
453 // to phase ordering between legalized code and the dag combiner. This
454 // probably means that we need to integrate dag combiner and legalizer
455 // together.
456 // We generally can't do this one for long doubles.
457 SDValue Chain = ST->getChain();
458 SDValue Ptr = ST->getBasePtr();
459 SDValue Value = ST->getValue();
460 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
461 AAMDNodes AAInfo = ST->getAAInfo();
462 SDLoc dl(ST);
463
464 // Don't optimise TargetConstantFP
465 if (Value.getOpcode() == ISD::TargetConstantFP)
466 return SDValue();
467
468 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
469 if (CFP->getValueType(0) == MVT::f32 &&
470 TLI.isTypeLegal(MVT::i32)) {
471 SDValue Con = DAG.getConstant(CFP->getValueAPF().
472 bitcastToAPInt().zextOrTrunc(32),
473 SDLoc(CFP), MVT::i32);
474 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
475 ST->getOriginalAlign(), MMOFlags, AAInfo);
476 }
477
478 if (CFP->getValueType(0) == MVT::f64 &&
479 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
480 // If this target supports 64-bit registers, do a single 64-bit store.
481 if (TLI.isTypeLegal(MVT::i64)) {
482 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
483 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
484 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
485 ST->getOriginalAlign(), MMOFlags, AAInfo);
486 }
487
488 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
489 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
490 // stores. If the target supports neither 32- nor 64-bits, this
491 // xform is certainly not worth it.
492 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
493 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
494 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
495 if (DAG.getDataLayout().isBigEndian())
496 std::swap(Lo, Hi);
497
498 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
499 ST->getOriginalAlign(), MMOFlags, AAInfo);
500 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
501 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
502 ST->getPointerInfo().getWithOffset(4),
503 ST->getOriginalAlign(), MMOFlags, AAInfo);
504
505 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
506 }
507 }
508 }
509 return SDValue();
510}
511
512void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
513 StoreSDNode *ST = cast<StoreSDNode>(Node);
514 SDValue Chain = ST->getChain();
515 SDValue Ptr = ST->getBasePtr();
516 SDLoc dl(Node);
517
518 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
519 AAMDNodes AAInfo = ST->getAAInfo();
520
521 if (!ST->isTruncatingStore()) {
522 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
523 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
524 ReplaceNode(ST, OptStore);
525 return;
526 }
527
528 SDValue Value = ST->getValue();
529 MVT VT = Value.getSimpleValueType();
530 switch (TLI.getOperationAction(ISD::STORE, VT)) {
531 default: llvm_unreachable("This action is not supported yet!");
532 case TargetLowering::Legal: {
533 // If this is an unaligned store and the target doesn't support it,
534 // expand it.
535 EVT MemVT = ST->getMemoryVT();
536 const DataLayout &DL = DAG.getDataLayout();
537 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
538 *ST->getMemOperand())) {
539 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
540 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
541 ReplaceNode(SDValue(ST, 0), Result);
542 } else
543 LLVM_DEBUG(dbgs() << "Legal store\n");
544 break;
545 }
546 case TargetLowering::Custom: {
547 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
548 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
549 if (Res && Res != SDValue(Node, 0))
550 ReplaceNode(SDValue(Node, 0), Res);
551 return;
552 }
553 case TargetLowering::Promote: {
554 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
555 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
556 "Can only promote stores to same size type");
557 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
558 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
559 ST->getOriginalAlign(), MMOFlags, AAInfo);
560 ReplaceNode(SDValue(Node, 0), Result);
561 break;
562 }
563 }
564 return;
565 }
566
567 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
568 SDValue Value = ST->getValue();
569 EVT StVT = ST->getMemoryVT();
570 TypeSize StWidth = StVT.getSizeInBits();
571 TypeSize StSize = StVT.getStoreSizeInBits();
572 auto &DL = DAG.getDataLayout();
573
574 if (StWidth != StSize) {
575 // Promote to a byte-sized store with upper bits zero if not
576 // storing an integral number of bytes. For example, promote
577 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
578 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
579 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
581 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
582 ST->getOriginalAlign(), MMOFlags, AAInfo);
583 ReplaceNode(SDValue(Node, 0), Result);
584 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
585 // If not storing a power-of-2 number of bits, expand as two stores.
586 assert(!StVT.isVector() && "Unsupported truncstore!");
587 unsigned StWidthBits = StWidth.getFixedValue();
588 unsigned LogStWidth = Log2_32(StWidthBits);
589 assert(LogStWidth < 32);
590 unsigned RoundWidth = 1 << LogStWidth;
591 assert(RoundWidth < StWidthBits);
592 unsigned ExtraWidth = StWidthBits - RoundWidth;
593 assert(ExtraWidth < RoundWidth);
594 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
595 "Store size not an integral number of bytes!");
596 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
597 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
598 SDValue Lo, Hi;
599 unsigned IncrementSize;
600
601 if (DL.isLittleEndian()) {
602 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
603 // Store the bottom RoundWidth bits.
604 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
605 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
606
607 // Store the remaining ExtraWidth bits.
608 IncrementSize = RoundWidth / 8;
609 Ptr =
610 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
611 Hi = DAG.getNode(
612 ISD::SRL, dl, Value.getValueType(), Value,
613 DAG.getConstant(RoundWidth, dl,
614 TLI.getShiftAmountTy(Value.getValueType(), DL)));
615 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
616 ST->getPointerInfo().getWithOffset(IncrementSize),
617 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
618 } else {
619 // Big endian - avoid unaligned stores.
620 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
621 // Store the top RoundWidth bits.
622 Hi = DAG.getNode(
623 ISD::SRL, dl, Value.getValueType(), Value,
624 DAG.getConstant(ExtraWidth, dl,
625 TLI.getShiftAmountTy(Value.getValueType(), DL)));
626 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
627 ST->getOriginalAlign(), MMOFlags, AAInfo);
628
629 // Store the remaining ExtraWidth bits.
630 IncrementSize = RoundWidth / 8;
631 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
632 DAG.getConstant(IncrementSize, dl,
633 Ptr.getValueType()));
634 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
635 ST->getPointerInfo().getWithOffset(IncrementSize),
636 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
637 }
638
639 // The order of the stores doesn't matter.
640 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
641 ReplaceNode(SDValue(Node, 0), Result);
642 } else {
643 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
644 default: llvm_unreachable("This action is not supported yet!");
645 case TargetLowering::Legal: {
646 EVT MemVT = ST->getMemoryVT();
647 // If this is an unaligned store and the target doesn't support it,
648 // expand it.
649 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
650 *ST->getMemOperand())) {
651 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
652 ReplaceNode(SDValue(ST, 0), Result);
653 }
654 break;
655 }
656 case TargetLowering::Custom: {
657 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
658 if (Res && Res != SDValue(Node, 0))
659 ReplaceNode(SDValue(Node, 0), Res);
660 return;
661 }
662 case TargetLowering::Expand:
663 assert(!StVT.isVector() &&
664 "Vector Stores are handled in LegalizeVectorOps");
665
667
668 // TRUNCSTORE:i16 i32 -> STORE i16
669 if (TLI.isTypeLegal(StVT)) {
670 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
671 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
672 ST->getOriginalAlign(), MMOFlags, AAInfo);
673 } else {
674 // The in-memory type isn't legal. Truncate to the type it would promote
675 // to, and then do a truncstore.
676 Value = DAG.getNode(ISD::TRUNCATE, dl,
677 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
678 Value);
679 Result =
680 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
681 ST->getOriginalAlign(), MMOFlags, AAInfo);
682 }
683
684 ReplaceNode(SDValue(Node, 0), Result);
685 break;
686 }
687 }
688}
689
690void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
691 LoadSDNode *LD = cast<LoadSDNode>(Node);
692 SDValue Chain = LD->getChain(); // The chain.
693 SDValue Ptr = LD->getBasePtr(); // The base pointer.
694 SDValue Value; // The value returned by the load op.
695 SDLoc dl(Node);
696
697 ISD::LoadExtType ExtType = LD->getExtensionType();
698 if (ExtType == ISD::NON_EXTLOAD) {
699 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
700 MVT VT = Node->getSimpleValueType(0);
701 SDValue RVal = SDValue(Node, 0);
702 SDValue RChain = SDValue(Node, 1);
703
704 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
705 default: llvm_unreachable("This action is not supported yet!");
706 case TargetLowering::Legal: {
707 EVT MemVT = LD->getMemoryVT();
708 const DataLayout &DL = DAG.getDataLayout();
709 // If this is an unaligned load and the target doesn't support it,
710 // expand it.
711 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
712 *LD->getMemOperand())) {
713 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
714 }
715 break;
716 }
717 case TargetLowering::Custom:
718 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
719 RVal = Res;
720 RChain = Res.getValue(1);
721 }
722 break;
723
724 case TargetLowering::Promote: {
725 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
726 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
727 "Can only promote loads to same size type");
728
729 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
730 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
731 RChain = Res.getValue(1);
732 break;
733 }
734 }
735 if (RChain.getNode() != Node) {
736 assert(RVal.getNode() != Node && "Load must be completely replaced");
737 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
738 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
739 if (UpdatedNodes) {
740 UpdatedNodes->insert(RVal.getNode());
741 UpdatedNodes->insert(RChain.getNode());
742 }
743 ReplacedNode(Node);
744 }
745 return;
746 }
747
748 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
749 EVT SrcVT = LD->getMemoryVT();
750 TypeSize SrcWidth = SrcVT.getSizeInBits();
751 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
752 AAMDNodes AAInfo = LD->getAAInfo();
753
754 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
755 // Some targets pretend to have an i1 loading operation, and actually
756 // load an i8. This trick is correct for ZEXTLOAD because the top 7
757 // bits are guaranteed to be zero; it helps the optimizers understand
758 // that these bits are zero. It is also useful for EXTLOAD, since it
759 // tells the optimizers that those bits are undefined. It would be
760 // nice to have an effective generic way of getting these benefits...
761 // Until such a way is found, don't insist on promoting i1 here.
762 (SrcVT != MVT::i1 ||
763 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
764 TargetLowering::Promote)) {
765 // Promote to a byte-sized load if not loading an integral number of
766 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
767 unsigned NewWidth = SrcVT.getStoreSizeInBits();
768 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
769 SDValue Ch;
770
771 // The extra bits are guaranteed to be zero, since we stored them that
772 // way. A zext load from NVT thus automatically gives zext from SrcVT.
773
774 ISD::LoadExtType NewExtType =
776
777 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
778 Chain, Ptr, LD->getPointerInfo(), NVT,
779 LD->getOriginalAlign(), MMOFlags, AAInfo);
780
781 Ch = Result.getValue(1); // The chain.
782
783 if (ExtType == ISD::SEXTLOAD)
784 // Having the top bits zero doesn't help when sign extending.
785 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
786 Result.getValueType(),
787 Result, DAG.getValueType(SrcVT));
788 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
789 // All the top bits are guaranteed to be zero - inform the optimizers.
790 Result = DAG.getNode(ISD::AssertZext, dl,
791 Result.getValueType(), Result,
792 DAG.getValueType(SrcVT));
793
794 Value = Result;
795 Chain = Ch;
796 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
797 // If not loading a power-of-2 number of bits, expand as two loads.
798 assert(!SrcVT.isVector() && "Unsupported extload!");
799 unsigned SrcWidthBits = SrcWidth.getFixedValue();
800 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
801 assert(LogSrcWidth < 32);
802 unsigned RoundWidth = 1 << LogSrcWidth;
803 assert(RoundWidth < SrcWidthBits);
804 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
805 assert(ExtraWidth < RoundWidth);
806 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
807 "Load size not an integral number of bytes!");
808 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
809 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
810 SDValue Lo, Hi, Ch;
811 unsigned IncrementSize;
812 auto &DL = DAG.getDataLayout();
813
814 if (DL.isLittleEndian()) {
815 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
816 // Load the bottom RoundWidth bits.
817 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
818 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
819 MMOFlags, AAInfo);
820
821 // Load the remaining ExtraWidth bits.
822 IncrementSize = RoundWidth / 8;
823 Ptr =
824 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
825 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
826 LD->getPointerInfo().getWithOffset(IncrementSize),
827 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
828
829 // Build a factor node to remember that this load is independent of
830 // the other one.
831 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
832 Hi.getValue(1));
833
834 // Move the top bits to the right place.
835 Hi = DAG.getNode(
836 ISD::SHL, dl, Hi.getValueType(), Hi,
837 DAG.getConstant(RoundWidth, dl,
838 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
839
840 // Join the hi and lo parts.
841 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
842 } else {
843 // Big endian - avoid unaligned loads.
844 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
845 // Load the top RoundWidth bits.
846 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
847 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
848 MMOFlags, AAInfo);
849
850 // Load the remaining ExtraWidth bits.
851 IncrementSize = RoundWidth / 8;
852 Ptr =
853 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
854 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
855 LD->getPointerInfo().getWithOffset(IncrementSize),
856 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
857
858 // Build a factor node to remember that this load is independent of
859 // the other one.
860 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
861 Hi.getValue(1));
862
863 // Move the top bits to the right place.
864 Hi = DAG.getNode(
865 ISD::SHL, dl, Hi.getValueType(), Hi,
866 DAG.getConstant(ExtraWidth, dl,
867 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
868
869 // Join the hi and lo parts.
870 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
871 }
872
873 Chain = Ch;
874 } else {
875 bool isCustom = false;
876 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
877 SrcVT.getSimpleVT())) {
878 default: llvm_unreachable("This action is not supported yet!");
879 case TargetLowering::Custom:
880 isCustom = true;
881 [[fallthrough]];
882 case TargetLowering::Legal:
883 Value = SDValue(Node, 0);
884 Chain = SDValue(Node, 1);
885
886 if (isCustom) {
887 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
888 Value = Res;
889 Chain = Res.getValue(1);
890 }
891 } else {
892 // If this is an unaligned load and the target doesn't support it,
893 // expand it.
894 EVT MemVT = LD->getMemoryVT();
895 const DataLayout &DL = DAG.getDataLayout();
896 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
897 *LD->getMemOperand())) {
898 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
899 }
900 }
901 break;
902
903 case TargetLowering::Expand: {
904 EVT DestVT = Node->getValueType(0);
905 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
906 // If the source type is not legal, see if there is a legal extload to
907 // an intermediate type that we can then extend further.
908 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
909 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
910 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
911 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
912 // If we are loading a legal type, this is a non-extload followed by a
913 // full extend.
914 ISD::LoadExtType MidExtType =
915 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
916
917 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
918 SrcVT, LD->getMemOperand());
919 unsigned ExtendOp =
921 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
922 Chain = Load.getValue(1);
923 break;
924 }
925
926 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
927 // normal undefined upper bits behavior to allow using an in-reg extend
928 // with the illegal FP type, so load as an integer and do the
929 // from-integer conversion.
930 EVT SVT = SrcVT.getScalarType();
931 if (SVT == MVT::f16 || SVT == MVT::bf16) {
932 EVT ISrcVT = SrcVT.changeTypeToInteger();
933 EVT IDestVT = DestVT.changeTypeToInteger();
934 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
935
936 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
937 Ptr, ISrcVT, LD->getMemOperand());
938 Value =
939 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
940 dl, DestVT, Result);
941 Chain = Result.getValue(1);
942 break;
943 }
944 }
945
946 assert(!SrcVT.isVector() &&
947 "Vector Loads are handled in LegalizeVectorOps");
948
949 // FIXME: This does not work for vectors on most targets. Sign-
950 // and zero-extend operations are currently folded into extending
951 // loads, whether they are legal or not, and then we end up here
952 // without any support for legalizing them.
953 assert(ExtType != ISD::EXTLOAD &&
954 "EXTLOAD should always be supported!");
955 // Turn the unsupported load into an EXTLOAD followed by an
956 // explicit zero/sign extend inreg.
957 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
958 Node->getValueType(0),
959 Chain, Ptr, SrcVT,
960 LD->getMemOperand());
961 SDValue ValRes;
962 if (ExtType == ISD::SEXTLOAD)
963 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
964 Result.getValueType(),
965 Result, DAG.getValueType(SrcVT));
966 else
967 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
968 Value = ValRes;
969 Chain = Result.getValue(1);
970 break;
971 }
972 }
973 }
974
975 // Since loads produce two values, make sure to remember that we legalized
976 // both of them.
977 if (Chain.getNode() != Node) {
978 assert(Value.getNode() != Node && "Load must be completely replaced");
979 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
980 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
981 if (UpdatedNodes) {
982 UpdatedNodes->insert(Value.getNode());
983 UpdatedNodes->insert(Chain.getNode());
984 }
985 ReplacedNode(Node);
986 }
987}
988
989/// Return a legal replacement for the given operation, with all legal operands.
990void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
991 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
992
993 // Allow illegal target nodes and illegal registers.
994 if (Node->getOpcode() == ISD::TargetConstant ||
995 Node->getOpcode() == ISD::Register)
996 return;
997
998#ifndef NDEBUG
999 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1000 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
1001 TargetLowering::TypeLegal &&
1002 "Unexpected illegal type!");
1003
1004 for (const SDValue &Op : Node->op_values())
1005 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
1006 TargetLowering::TypeLegal ||
1007 Op.getOpcode() == ISD::TargetConstant ||
1008 Op.getOpcode() == ISD::Register) &&
1009 "Unexpected illegal type!");
1010#endif
1011
1012 // Figure out the correct action; the way to query this varies by opcode
1013 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1014 bool SimpleFinishLegalizing = true;
1015 switch (Node->getOpcode()) {
1019 case ISD::STACKSAVE:
1020 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1021 break;
1023 Action = TLI.getOperationAction(Node->getOpcode(),
1024 Node->getValueType(0));
1025 break;
1026 case ISD::VAARG:
1027 Action = TLI.getOperationAction(Node->getOpcode(),
1028 Node->getValueType(0));
1029 if (Action != TargetLowering::Promote)
1030 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1031 break;
1032 case ISD::SET_FPENV:
1033 case ISD::SET_FPMODE:
1034 Action = TLI.getOperationAction(Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1036 break;
1037 case ISD::FP_TO_FP16:
1038 case ISD::FP_TO_BF16:
1039 case ISD::SINT_TO_FP:
1040 case ISD::UINT_TO_FP:
1042 case ISD::LROUND:
1043 case ISD::LLROUND:
1044 case ISD::LRINT:
1045 case ISD::LLRINT:
1046 Action = TLI.getOperationAction(Node->getOpcode(),
1047 Node->getOperand(0).getValueType());
1048 break;
1053 case ISD::STRICT_LRINT:
1054 case ISD::STRICT_LLRINT:
1055 case ISD::STRICT_LROUND:
1057 // These pseudo-ops are the same as the other STRICT_ ops except
1058 // they are registered with setOperationAction() using the input type
1059 // instead of the output type.
1060 Action = TLI.getOperationAction(Node->getOpcode(),
1061 Node->getOperand(1).getValueType());
1062 break;
1064 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1065 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1066 break;
1067 }
1068 case ISD::ATOMIC_STORE:
1069 Action = TLI.getOperationAction(Node->getOpcode(),
1070 Node->getOperand(1).getValueType());
1071 break;
1072 case ISD::SELECT_CC:
1073 case ISD::STRICT_FSETCC:
1075 case ISD::SETCC:
1076 case ISD::SETCCCARRY:
1077 case ISD::VP_SETCC:
1078 case ISD::BR_CC: {
1079 unsigned Opc = Node->getOpcode();
1080 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1081 : Opc == ISD::STRICT_FSETCC ? 3
1082 : Opc == ISD::STRICT_FSETCCS ? 3
1083 : Opc == ISD::SETCCCARRY ? 3
1084 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1085 : 1;
1086 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1087 : Opc == ISD::STRICT_FSETCC ? 1
1088 : Opc == ISD::STRICT_FSETCCS ? 1
1089 : 0;
1090 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1091 ISD::CondCode CCCode =
1092 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1093 Action = TLI.getCondCodeAction(CCCode, OpVT);
1094 if (Action == TargetLowering::Legal) {
1095 if (Node->getOpcode() == ISD::SELECT_CC)
1096 Action = TLI.getOperationAction(Node->getOpcode(),
1097 Node->getValueType(0));
1098 else
1099 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1100 }
1101 break;
1102 }
1103 case ISD::LOAD:
1104 case ISD::STORE:
1105 // FIXME: Model these properly. LOAD and STORE are complicated, and
1106 // STORE expects the unlegalized operand in some cases.
1107 SimpleFinishLegalizing = false;
1108 break;
1109 case ISD::CALLSEQ_START:
1110 case ISD::CALLSEQ_END:
1111 // FIXME: This shouldn't be necessary. These nodes have special properties
1112 // dealing with the recursive nature of legalization. Removing this
1113 // special case should be done as part of making LegalizeDAG non-recursive.
1114 SimpleFinishLegalizing = false;
1115 break;
1117 case ISD::GET_ROUNDING:
1118 case ISD::MERGE_VALUES:
1119 case ISD::EH_RETURN:
1121 case ISD::EH_DWARF_CFA:
1125 // These operations lie about being legal: when they claim to be legal,
1126 // they should actually be expanded.
1127 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1128 if (Action == TargetLowering::Legal)
1129 Action = TargetLowering::Expand;
1130 break;
1133 case ISD::FRAMEADDR:
1134 case ISD::RETURNADDR:
1136 case ISD::SPONENTRY:
1137 // These operations lie about being legal: when they claim to be legal,
1138 // they should actually be custom-lowered.
1139 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1140 if (Action == TargetLowering::Legal)
1141 Action = TargetLowering::Custom;
1142 break;
1145 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1146 // legalization might have expanded that to several smaller types.
1147 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1148 break;
1149 case ISD::READ_REGISTER:
1151 // Named register is legal in the DAG, but blocked by register name
1152 // selection if not implemented by target (to chose the correct register)
1153 // They'll be converted to Copy(To/From)Reg.
1154 Action = TargetLowering::Legal;
1155 break;
1156 case ISD::UBSANTRAP:
1157 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1158 if (Action == TargetLowering::Expand) {
1159 // replace ISD::UBSANTRAP with ISD::TRAP
1160 SDValue NewVal;
1161 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1162 Node->getOperand(0));
1163 ReplaceNode(Node, NewVal.getNode());
1164 LegalizeOp(NewVal.getNode());
1165 return;
1166 }
1167 break;
1168 case ISD::DEBUGTRAP:
1169 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1170 if (Action == TargetLowering::Expand) {
1171 // replace ISD::DEBUGTRAP with ISD::TRAP
1172 SDValue NewVal;
1173 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1174 Node->getOperand(0));
1175 ReplaceNode(Node, NewVal.getNode());
1176 LegalizeOp(NewVal.getNode());
1177 return;
1178 }
1179 break;
1180 case ISD::SADDSAT:
1181 case ISD::UADDSAT:
1182 case ISD::SSUBSAT:
1183 case ISD::USUBSAT:
1184 case ISD::SSHLSAT:
1185 case ISD::USHLSAT:
1188 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1189 break;
1190 case ISD::SMULFIX:
1191 case ISD::SMULFIXSAT:
1192 case ISD::UMULFIX:
1193 case ISD::UMULFIXSAT:
1194 case ISD::SDIVFIX:
1195 case ISD::SDIVFIXSAT:
1196 case ISD::UDIVFIX:
1197 case ISD::UDIVFIXSAT: {
1198 unsigned Scale = Node->getConstantOperandVal(2);
1199 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1200 Node->getValueType(0), Scale);
1201 break;
1202 }
1203 case ISD::MSCATTER:
1204 Action = TLI.getOperationAction(Node->getOpcode(),
1205 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1206 break;
1207 case ISD::MSTORE:
1208 Action = TLI.getOperationAction(Node->getOpcode(),
1209 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1210 break;
1211 case ISD::VP_SCATTER:
1212 Action = TLI.getOperationAction(
1213 Node->getOpcode(),
1214 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1215 break;
1216 case ISD::VP_STORE:
1217 Action = TLI.getOperationAction(
1218 Node->getOpcode(),
1219 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1220 break;
1221 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1222 Action = TLI.getOperationAction(
1223 Node->getOpcode(),
1224 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1225 break;
1228 case ISD::VECREDUCE_ADD:
1229 case ISD::VECREDUCE_MUL:
1230 case ISD::VECREDUCE_AND:
1231 case ISD::VECREDUCE_OR:
1232 case ISD::VECREDUCE_XOR:
1241 case ISD::IS_FPCLASS:
1242 Action = TLI.getOperationAction(
1243 Node->getOpcode(), Node->getOperand(0).getValueType());
1244 break;
1247 case ISD::VP_REDUCE_FADD:
1248 case ISD::VP_REDUCE_FMUL:
1249 case ISD::VP_REDUCE_ADD:
1250 case ISD::VP_REDUCE_MUL:
1251 case ISD::VP_REDUCE_AND:
1252 case ISD::VP_REDUCE_OR:
1253 case ISD::VP_REDUCE_XOR:
1254 case ISD::VP_REDUCE_SMAX:
1255 case ISD::VP_REDUCE_SMIN:
1256 case ISD::VP_REDUCE_UMAX:
1257 case ISD::VP_REDUCE_UMIN:
1258 case ISD::VP_REDUCE_FMAX:
1259 case ISD::VP_REDUCE_FMIN:
1260 case ISD::VP_REDUCE_SEQ_FADD:
1261 case ISD::VP_REDUCE_SEQ_FMUL:
1262 Action = TLI.getOperationAction(
1263 Node->getOpcode(), Node->getOperand(1).getValueType());
1264 break;
1265 default:
1266 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1267 Action = TLI.getCustomOperationAction(*Node);
1268 } else {
1269 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1270 }
1271 break;
1272 }
1273
1274 if (SimpleFinishLegalizing) {
1275 SDNode *NewNode = Node;
1276 switch (Node->getOpcode()) {
1277 default: break;
1278 case ISD::SHL:
1279 case ISD::SRL:
1280 case ISD::SRA:
1281 case ISD::ROTL:
1282 case ISD::ROTR: {
1283 // Legalizing shifts/rotates requires adjusting the shift amount
1284 // to the appropriate width.
1285 SDValue Op0 = Node->getOperand(0);
1286 SDValue Op1 = Node->getOperand(1);
1287 if (!Op1.getValueType().isVector()) {
1288 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1289 // The getShiftAmountOperand() may create a new operand node or
1290 // return the existing one. If new operand is created we need
1291 // to update the parent node.
1292 // Do not try to legalize SAO here! It will be automatically legalized
1293 // in the next round.
1294 if (SAO != Op1)
1295 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1296 }
1297 }
1298 break;
1299 case ISD::FSHL:
1300 case ISD::FSHR:
1301 case ISD::SRL_PARTS:
1302 case ISD::SRA_PARTS:
1303 case ISD::SHL_PARTS: {
1304 // Legalizing shifts/rotates requires adjusting the shift amount
1305 // to the appropriate width.
1306 SDValue Op0 = Node->getOperand(0);
1307 SDValue Op1 = Node->getOperand(1);
1308 SDValue Op2 = Node->getOperand(2);
1309 if (!Op2.getValueType().isVector()) {
1310 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1311 // The getShiftAmountOperand() may create a new operand node or
1312 // return the existing one. If new operand is created we need
1313 // to update the parent node.
1314 if (SAO != Op2)
1315 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1316 }
1317 break;
1318 }
1319 }
1320
1321 if (NewNode != Node) {
1322 ReplaceNode(Node, NewNode);
1323 Node = NewNode;
1324 }
1325 switch (Action) {
1326 case TargetLowering::Legal:
1327 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1328 return;
1329 case TargetLowering::Custom:
1330 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1331 // FIXME: The handling for custom lowering with multiple results is
1332 // a complete mess.
1333 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1334 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1335 return;
1336
1337 if (Node->getNumValues() == 1) {
1338 // Verify the new types match the original. Glue is waived because
1339 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1340 assert((Res.getValueType() == Node->getValueType(0) ||
1341 Node->getValueType(0) == MVT::Glue) &&
1342 "Type mismatch for custom legalized operation");
1343 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1344 // We can just directly replace this node with the lowered value.
1345 ReplaceNode(SDValue(Node, 0), Res);
1346 return;
1347 }
1348
1349 SmallVector<SDValue, 8> ResultVals;
1350 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1351 // Verify the new types match the original. Glue is waived because
1352 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1353 assert((Res->getValueType(i) == Node->getValueType(i) ||
1354 Node->getValueType(i) == MVT::Glue) &&
1355 "Type mismatch for custom legalized operation");
1356 ResultVals.push_back(Res.getValue(i));
1357 }
1358 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1359 ReplaceNode(Node, ResultVals.data());
1360 return;
1361 }
1362 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1363 [[fallthrough]];
1364 case TargetLowering::Expand:
1365 if (ExpandNode(Node))
1366 return;
1367 [[fallthrough]];
1368 case TargetLowering::LibCall:
1369 ConvertNodeToLibcall(Node);
1370 return;
1371 case TargetLowering::Promote:
1372 PromoteNode(Node);
1373 return;
1374 }
1375 }
1376
1377 switch (Node->getOpcode()) {
1378 default:
1379#ifndef NDEBUG
1380 dbgs() << "NODE: ";
1381 Node->dump( &DAG);
1382 dbgs() << "\n";
1383#endif
1384 llvm_unreachable("Do not know how to legalize this operator!");
1385
1386 case ISD::CALLSEQ_START:
1387 case ISD::CALLSEQ_END:
1388 break;
1389 case ISD::LOAD:
1390 return LegalizeLoadOps(Node);
1391 case ISD::STORE:
1392 return LegalizeStoreOps(Node);
1393 }
1394}
1395
1396SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1397 SDValue Vec = Op.getOperand(0);
1398 SDValue Idx = Op.getOperand(1);
1399 SDLoc dl(Op);
1400
1401 // Before we generate a new store to a temporary stack slot, see if there is
1402 // already one that we can use. There often is because when we scalarize
1403 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1404 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1405 // the vector. If all are expanded here, we don't want one store per vector
1406 // element.
1407
1408 // Caches for hasPredecessorHelper
1411 Visited.insert(Op.getNode());
1412 Worklist.push_back(Idx.getNode());
1413 SDValue StackPtr, Ch;
1414 for (SDNode *User : Vec.getNode()->uses()) {
1415 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1416 if (ST->isIndexed() || ST->isTruncatingStore() ||
1417 ST->getValue() != Vec)
1418 continue;
1419
1420 // Make sure that nothing else could have stored into the destination of
1421 // this store.
1422 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1423 continue;
1424
1425 // If the index is dependent on the store we will introduce a cycle when
1426 // creating the load (the load uses the index, and by replacing the chain
1427 // we will make the index dependent on the load). Also, the store might be
1428 // dependent on the extractelement and introduce a cycle when creating
1429 // the load.
1430 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1431 ST->hasPredecessor(Op.getNode()))
1432 continue;
1433
1434 StackPtr = ST->getBasePtr();
1435 Ch = SDValue(ST, 0);
1436 break;
1437 }
1438 }
1439
1440 EVT VecVT = Vec.getValueType();
1441
1442 if (!Ch.getNode()) {
1443 // Store the value to a temporary stack slot, then LOAD the returned part.
1444 StackPtr = DAG.CreateStackTemporary(VecVT);
1446 StackPtr, DAG.getMachineFunction(), VecVT.isScalableVector());
1447 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1448 }
1449
1450 SDValue NewLoad;
1451 Align ElementAlignment =
1452 std::min(cast<StoreSDNode>(Ch)->getAlign(),
1453 DAG.getDataLayout().getPrefTypeAlign(
1454 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1455
1456 if (Op.getValueType().isVector()) {
1457 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1458 Op.getValueType(), Idx);
1459 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1460 MachinePointerInfo(), ElementAlignment);
1461 } else {
1462 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1463 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1465 ElementAlignment);
1466 }
1467
1468 // Replace the chain going out of the store, by the one out of the load.
1469 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1470
1471 // We introduced a cycle though, so update the loads operands, making sure
1472 // to use the original store's chain as an incoming chain.
1473 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1474 NewLoad->op_end());
1475 NewLoadOperands[0] = Ch;
1476 NewLoad =
1477 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1478 return NewLoad;
1479}
1480
1481SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1482 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1483
1484 SDValue Vec = Op.getOperand(0);
1485 SDValue Part = Op.getOperand(1);
1486 SDValue Idx = Op.getOperand(2);
1487 SDLoc dl(Op);
1488
1489 // Store the value to a temporary stack slot, then LOAD the returned part.
1490 EVT VecVT = Vec.getValueType();
1491 EVT SubVecVT = Part.getValueType();
1492 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1493 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1494 MachinePointerInfo PtrInfo =
1495 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1496
1497 // First store the whole vector.
1498 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1499
1500 // Then store the inserted part.
1501 SDValue SubStackPtr =
1502 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1503
1504 // Store the subvector.
1505 Ch = DAG.getStore(
1506 Ch, dl, Part, SubStackPtr,
1507 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1508
1509 // Finally, load the updated vector.
1510 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1511}
1512
1513SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1514 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1515 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1516 "Unexpected opcode!");
1517
1518 // We can't handle this case efficiently. Allocate a sufficiently
1519 // aligned object on the stack, store each operand into it, then load
1520 // the result as a vector.
1521 // Create the stack frame object.
1522 EVT VT = Node->getValueType(0);
1523 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1524 : Node->getOperand(0).getValueType();
1525 SDLoc dl(Node);
1526 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1527 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1528 MachinePointerInfo PtrInfo =
1529 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1530
1531 // Emit a store of each element to the stack slot.
1533 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1534 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1535
1536 // If the destination vector element type of a BUILD_VECTOR is narrower than
1537 // the source element type, only store the bits necessary.
1538 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1539 MemVT.bitsLT(Node->getOperand(0).getValueType());
1540
1541 // Store (in the right endianness) the elements to memory.
1542 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1543 // Ignore undef elements.
1544 if (Node->getOperand(i).isUndef()) continue;
1545
1546 unsigned Offset = TypeByteSize*i;
1547
1548 SDValue Idx =
1549 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1550
1551 if (Truncate)
1552 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1553 Node->getOperand(i), Idx,
1554 PtrInfo.getWithOffset(Offset), MemVT));
1555 else
1556 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1557 Idx, PtrInfo.getWithOffset(Offset)));
1558 }
1559
1560 SDValue StoreChain;
1561 if (!Stores.empty()) // Not all undef elements?
1562 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1563 else
1564 StoreChain = DAG.getEntryNode();
1565
1566 // Result is a load from the stack slot.
1567 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1568}
1569
1570/// Bitcast a floating-point value to an integer value. Only bitcast the part
1571/// containing the sign bit if the target has no integer value capable of
1572/// holding all bits of the floating-point value.
1573void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1574 const SDLoc &DL,
1575 SDValue Value) const {
1576 EVT FloatVT = Value.getValueType();
1577 unsigned NumBits = FloatVT.getScalarSizeInBits();
1578 State.FloatVT = FloatVT;
1579 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1580 // Convert to an integer of the same size.
1581 if (TLI.isTypeLegal(IVT)) {
1582 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1583 State.SignMask = APInt::getSignMask(NumBits);
1584 State.SignBit = NumBits - 1;
1585 return;
1586 }
1587
1588 auto &DataLayout = DAG.getDataLayout();
1589 // Store the float to memory, then load the sign part out as an integer.
1590 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1591 // First create a temporary that is aligned for both the load and store.
1592 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1593 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1594 // Then store the float to it.
1595 State.FloatPtr = StackPtr;
1596 MachineFunction &MF = DAG.getMachineFunction();
1597 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1598 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1599 State.FloatPointerInfo);
1600
1601 SDValue IntPtr;
1602 if (DataLayout.isBigEndian()) {
1603 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1604 // Load out a legal integer with the same sign bit as the float.
1605 IntPtr = StackPtr;
1606 State.IntPointerInfo = State.FloatPointerInfo;
1607 } else {
1608 // Advance the pointer so that the loaded byte will contain the sign bit.
1609 unsigned ByteOffset = (NumBits / 8) - 1;
1610 IntPtr =
1611 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1612 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1613 ByteOffset);
1614 }
1615
1616 State.IntPtr = IntPtr;
1617 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1618 State.IntPointerInfo, MVT::i8);
1619 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1620 State.SignBit = 7;
1621}
1622
1623/// Replace the integer value produced by getSignAsIntValue() with a new value
1624/// and cast the result back to a floating-point type.
1625SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1626 const SDLoc &DL,
1627 SDValue NewIntValue) const {
1628 if (!State.Chain)
1629 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1630
1631 // Override the part containing the sign bit in the value stored on the stack.
1632 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1633 State.IntPointerInfo, MVT::i8);
1634 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1635 State.FloatPointerInfo);
1636}
1637
1638SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1639 SDLoc DL(Node);
1640 SDValue Mag = Node->getOperand(0);
1641 SDValue Sign = Node->getOperand(1);
1642
1643 // Get sign bit into an integer value.
1644 FloatSignAsInt SignAsInt;
1645 getSignAsIntValue(SignAsInt, DL, Sign);
1646
1647 EVT IntVT = SignAsInt.IntValue.getValueType();
1648 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1649 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1650 SignMask);
1651
1652 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1653 EVT FloatVT = Mag.getValueType();
1654 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1655 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1656 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1657 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1658 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1659 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1660 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1661 }
1662
1663 // Transform Mag value to integer, and clear the sign bit.
1664 FloatSignAsInt MagAsInt;
1665 getSignAsIntValue(MagAsInt, DL, Mag);
1666 EVT MagVT = MagAsInt.IntValue.getValueType();
1667 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1668 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1669 ClearSignMask);
1670
1671 // Get the signbit at the right position for MagAsInt.
1672 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1673 EVT ShiftVT = IntVT;
1674 if (SignBit.getScalarValueSizeInBits() <
1675 ClearedSign.getScalarValueSizeInBits()) {
1676 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1677 ShiftVT = MagVT;
1678 }
1679 if (ShiftAmount > 0) {
1680 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1681 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1682 } else if (ShiftAmount < 0) {
1683 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1684 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1685 }
1686 if (SignBit.getScalarValueSizeInBits() >
1687 ClearedSign.getScalarValueSizeInBits()) {
1688 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1689 }
1690
1691 // Store the part with the modified sign and convert back to float.
1692 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1693 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1694}
1695
1696SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1697 // Get the sign bit as an integer.
1698 SDLoc DL(Node);
1699 FloatSignAsInt SignAsInt;
1700 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1701 EVT IntVT = SignAsInt.IntValue.getValueType();
1702
1703 // Flip the sign.
1704 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1705 SDValue SignFlip =
1706 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1707
1708 // Convert back to float.
1709 return modifySignAsInt(SignAsInt, DL, SignFlip);
1710}
1711
1712SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1713 SDLoc DL(Node);
1714 SDValue Value = Node->getOperand(0);
1715
1716 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1717 EVT FloatVT = Value.getValueType();
1718 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1719 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1720 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1721 }
1722
1723 // Transform value to integer, clear the sign bit and transform back.
1724 FloatSignAsInt ValueAsInt;
1725 getSignAsIntValue(ValueAsInt, DL, Value);
1726 EVT IntVT = ValueAsInt.IntValue.getValueType();
1727 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1728 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1729 ClearSignMask);
1730 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1731}
1732
1733void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1735 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1736 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1737 " not tell us which reg is the stack pointer!");
1738 SDLoc dl(Node);
1739 EVT VT = Node->getValueType(0);
1740 SDValue Tmp1 = SDValue(Node, 0);
1741 SDValue Tmp2 = SDValue(Node, 1);
1742 SDValue Tmp3 = Node->getOperand(2);
1743 SDValue Chain = Tmp1.getOperand(0);
1744
1745 // Chain the dynamic stack allocation so that it doesn't modify the stack
1746 // pointer when other instructions are using the stack.
1747 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1748
1749 SDValue Size = Tmp2.getOperand(1);
1750 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1751 Chain = SP.getValue(1);
1752 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1753 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1754 unsigned Opc =
1757
1758 Align StackAlign = TFL->getStackAlign();
1759 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1760 if (Alignment > StackAlign)
1761 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1762 DAG.getConstant(-Alignment.value(), dl, VT));
1763 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1764
1765 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1766
1767 Results.push_back(Tmp1);
1768 Results.push_back(Tmp2);
1769}
1770
1771/// Emit a store/load combination to the stack. This stores
1772/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1773/// a load from the stack slot to DestVT, extending it if needed.
1774/// The resultant code need not be legal.
1775SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1776 EVT DestVT, const SDLoc &dl) {
1777 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1778}
1779
1780SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1781 EVT DestVT, const SDLoc &dl,
1782 SDValue Chain) {
1783 EVT SrcVT = SrcOp.getValueType();
1784 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1785 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1786
1787 // Don't convert with stack if the load/store is expensive.
1788 if ((SrcVT.bitsGT(SlotVT) &&
1789 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1790 (SlotVT.bitsLT(DestVT) &&
1791 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1792 return SDValue();
1793
1794 // Create the stack frame object.
1795 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1796 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1797 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1798
1799 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1800 int SPFI = StackPtrFI->getIndex();
1801 MachinePointerInfo PtrInfo =
1802 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1803
1804 // Emit a store to the stack slot. Use a truncstore if the input value is
1805 // later than DestVT.
1806 SDValue Store;
1807
1808 if (SrcVT.bitsGT(SlotVT))
1809 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1810 SlotVT, SrcAlign);
1811 else {
1812 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1813 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1814 }
1815
1816 // Result is a load from the stack slot.
1817 if (SlotVT.bitsEq(DestVT))
1818 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1819
1820 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1821 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1822 DestAlign);
1823}
1824
1825SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1826 SDLoc dl(Node);
1827 // Create a vector sized/aligned stack slot, store the value to element #0,
1828 // then load the whole vector back out.
1829 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1830
1831 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1832 int SPFI = StackPtrFI->getIndex();
1833
1834 SDValue Ch = DAG.getTruncStore(
1835 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1836 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1837 Node->getValueType(0).getVectorElementType());
1838 return DAG.getLoad(
1839 Node->getValueType(0), dl, Ch, StackPtr,
1840 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1841}
1842
1843static bool
1845 const TargetLowering &TLI, SDValue &Res) {
1846 unsigned NumElems = Node->getNumOperands();
1847 SDLoc dl(Node);
1848 EVT VT = Node->getValueType(0);
1849
1850 // Try to group the scalars into pairs, shuffle the pairs together, then
1851 // shuffle the pairs of pairs together, etc. until the vector has
1852 // been built. This will work only if all of the necessary shuffle masks
1853 // are legal.
1854
1855 // We do this in two phases; first to check the legality of the shuffles,
1856 // and next, assuming that all shuffles are legal, to create the new nodes.
1857 for (int Phase = 0; Phase < 2; ++Phase) {
1859 NewIntermedVals;
1860 for (unsigned i = 0; i < NumElems; ++i) {
1861 SDValue V = Node->getOperand(i);
1862 if (V.isUndef())
1863 continue;
1864
1865 SDValue Vec;
1866 if (Phase)
1867 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1868 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1869 }
1870
1871 while (IntermedVals.size() > 2) {
1872 NewIntermedVals.clear();
1873 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1874 // This vector and the next vector are shuffled together (simply to
1875 // append the one to the other).
1876 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1877
1878 SmallVector<int, 16> FinalIndices;
1879 FinalIndices.reserve(IntermedVals[i].second.size() +
1880 IntermedVals[i+1].second.size());
1881
1882 int k = 0;
1883 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1884 ++j, ++k) {
1885 ShuffleVec[k] = j;
1886 FinalIndices.push_back(IntermedVals[i].second[j]);
1887 }
1888 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1889 ++j, ++k) {
1890 ShuffleVec[k] = NumElems + j;
1891 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1892 }
1893
1894 SDValue Shuffle;
1895 if (Phase)
1896 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1897 IntermedVals[i+1].first,
1898 ShuffleVec);
1899 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1900 return false;
1901 NewIntermedVals.push_back(
1902 std::make_pair(Shuffle, std::move(FinalIndices)));
1903 }
1904
1905 // If we had an odd number of defined values, then append the last
1906 // element to the array of new vectors.
1907 if ((IntermedVals.size() & 1) != 0)
1908 NewIntermedVals.push_back(IntermedVals.back());
1909
1910 IntermedVals.swap(NewIntermedVals);
1911 }
1912
1913 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1914 "Invalid number of intermediate vectors");
1915 SDValue Vec1 = IntermedVals[0].first;
1916 SDValue Vec2;
1917 if (IntermedVals.size() > 1)
1918 Vec2 = IntermedVals[1].first;
1919 else if (Phase)
1920 Vec2 = DAG.getUNDEF(VT);
1921
1922 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1923 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1924 ShuffleVec[IntermedVals[0].second[i]] = i;
1925 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1926 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1927
1928 if (Phase)
1929 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1930 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1931 return false;
1932 }
1933
1934 return true;
1935}
1936
1937/// Expand a BUILD_VECTOR node on targets that don't
1938/// support the operation, but do support the resultant vector type.
1939SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1940 unsigned NumElems = Node->getNumOperands();
1941 SDValue Value1, Value2;
1942 SDLoc dl(Node);
1943 EVT VT = Node->getValueType(0);
1944 EVT OpVT = Node->getOperand(0).getValueType();
1945 EVT EltVT = VT.getVectorElementType();
1946
1947 // If the only non-undef value is the low element, turn this into a
1948 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1949 bool isOnlyLowElement = true;
1950 bool MoreThanTwoValues = false;
1951 bool isConstant = true;
1952 for (unsigned i = 0; i < NumElems; ++i) {
1953 SDValue V = Node->getOperand(i);
1954 if (V.isUndef())
1955 continue;
1956 if (i > 0)
1957 isOnlyLowElement = false;
1958 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1959 isConstant = false;
1960
1961 if (!Value1.getNode()) {
1962 Value1 = V;
1963 } else if (!Value2.getNode()) {
1964 if (V != Value1)
1965 Value2 = V;
1966 } else if (V != Value1 && V != Value2) {
1967 MoreThanTwoValues = true;
1968 }
1969 }
1970
1971 if (!Value1.getNode())
1972 return DAG.getUNDEF(VT);
1973
1974 if (isOnlyLowElement)
1975 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1976
1977 // If all elements are constants, create a load from the constant pool.
1978 if (isConstant) {
1980 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1981 if (ConstantFPSDNode *V =
1982 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1983 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1984 } else if (ConstantSDNode *V =
1985 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1986 if (OpVT==EltVT)
1987 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1988 else {
1989 // If OpVT and EltVT don't match, EltVT is not legal and the
1990 // element values have been promoted/truncated earlier. Undo this;
1991 // we don't want a v16i8 to become a v16i32 for example.
1992 const ConstantInt *CI = V->getConstantIntValue();
1993 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1994 CI->getZExtValue()));
1995 }
1996 } else {
1997 assert(Node->getOperand(i).isUndef());
1998 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1999 CV.push_back(UndefValue::get(OpNTy));
2000 }
2001 }
2003 SDValue CPIdx =
2004 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2005 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2006 return DAG.getLoad(
2007 VT, dl, DAG.getEntryNode(), CPIdx,
2008 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2009 Alignment);
2010 }
2011
2012 SmallSet<SDValue, 16> DefinedValues;
2013 for (unsigned i = 0; i < NumElems; ++i) {
2014 if (Node->getOperand(i).isUndef())
2015 continue;
2016 DefinedValues.insert(Node->getOperand(i));
2017 }
2018
2019 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
2020 if (!MoreThanTwoValues) {
2021 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2022 for (unsigned i = 0; i < NumElems; ++i) {
2023 SDValue V = Node->getOperand(i);
2024 if (V.isUndef())
2025 continue;
2026 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2027 }
2028 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2029 // Get the splatted value into the low element of a vector register.
2030 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2031 SDValue Vec2;
2032 if (Value2.getNode())
2033 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2034 else
2035 Vec2 = DAG.getUNDEF(VT);
2036
2037 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2038 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2039 }
2040 } else {
2041 SDValue Res;
2042 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2043 return Res;
2044 }
2045 }
2046
2047 // Otherwise, we can't handle this case efficiently.
2048 return ExpandVectorBuildThroughStack(Node);
2049}
2050
2051SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2052 SDLoc DL(Node);
2053 EVT VT = Node->getValueType(0);
2054 SDValue SplatVal = Node->getOperand(0);
2055
2056 return DAG.getSplatBuildVector(VT, DL, SplatVal);
2057}
2058
2059// Expand a node into a call to a libcall, returning the value as the first
2060// result and the chain as the second. If the result value does not fit into a
2061// register, return the lo part and set the hi part to the by-reg argument in
2062// the first. If it does fit into a single register, return the result and
2063// leave the Hi part unset.
2064std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2066 bool isSigned) {
2067 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2068 TLI.getPointerTy(DAG.getDataLayout()));
2069
2070 EVT RetVT = Node->getValueType(0);
2071 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2072
2073 // By default, the input chain to this libcall is the entry node of the
2074 // function. If the libcall is going to be emitted as a tail call then
2075 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2076 // node which is being folded has a non-entry input chain.
2077 SDValue InChain = DAG.getEntryNode();
2078
2079 // isTailCall may be true since the callee does not reference caller stack
2080 // frame. Check if it's in the right position and that the return types match.
2081 SDValue TCChain = InChain;
2082 const Function &F = DAG.getMachineFunction().getFunction();
2083 bool isTailCall =
2084 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2085 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2086 if (isTailCall)
2087 InChain = TCChain;
2088
2090 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2091 CLI.setDebugLoc(SDLoc(Node))
2092 .setChain(InChain)
2093 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2094 std::move(Args))
2095 .setTailCall(isTailCall)
2096 .setSExtResult(signExtend)
2097 .setZExtResult(!signExtend)
2098 .setIsPostTypeLegalization(true);
2099
2100 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2101
2102 if (!CallInfo.second.getNode()) {
2103 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2104 // It's a tailcall, return the chain (which is the DAG root).
2105 return {DAG.getRoot(), DAG.getRoot()};
2106 }
2107
2108 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2109 return CallInfo;
2110}
2111
2112std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2113 bool isSigned) {
2116 for (const SDValue &Op : Node->op_values()) {
2117 EVT ArgVT = Op.getValueType();
2118 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2119 Entry.Node = Op;
2120 Entry.Ty = ArgTy;
2121 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2122 Entry.IsZExt = !Entry.IsSExt;
2123 Args.push_back(Entry);
2124 }
2125
2126 return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2127}
2128
2129void SelectionDAGLegalize::ExpandFrexpLibCall(
2131 SDLoc dl(Node);
2132 EVT VT = Node->getValueType(0);
2133 EVT ExpVT = Node->getValueType(1);
2134
2135 SDValue FPOp = Node->getOperand(0);
2136
2137 EVT ArgVT = FPOp.getValueType();
2138 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2139
2141 FPArgEntry.Node = FPOp;
2142 FPArgEntry.Ty = ArgTy;
2143
2144 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2145 TargetLowering::ArgListEntry PtrArgEntry;
2146 PtrArgEntry.Node = StackSlot;
2147 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2148 DAG.getDataLayout().getAllocaAddrSpace());
2149
2150 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2151
2153 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2154
2155 // FIXME: Get type of int for libcall declaration and cast
2156
2157 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2158 auto PtrInfo =
2159 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2160
2161 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2162 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2163 LoadExp.getValue(1), DAG.getRoot());
2164 DAG.setRoot(OutputChain);
2165
2166 Results.push_back(Call);
2167 Results.push_back(LoadExp);
2168}
2169
2170void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2171 RTLIB::Libcall LC,
2173 if (LC == RTLIB::UNKNOWN_LIBCALL)
2174 llvm_unreachable("Can't create an unknown libcall!");
2175
2176 if (Node->isStrictFPOpcode()) {
2177 EVT RetVT = Node->getValueType(0);
2180 // FIXME: This doesn't support tail calls.
2181 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2182 Ops, CallOptions,
2183 SDLoc(Node),
2184 Node->getOperand(0));
2185 Results.push_back(Tmp.first);
2186 Results.push_back(Tmp.second);
2187 } else {
2188 SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2189 Results.push_back(Tmp);
2190 }
2191}
2192
2193/// Expand the node to a libcall based on the result type.
2194void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2195 RTLIB::Libcall Call_F32,
2196 RTLIB::Libcall Call_F64,
2197 RTLIB::Libcall Call_F80,
2198 RTLIB::Libcall Call_F128,
2199 RTLIB::Libcall Call_PPCF128,
2201 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2202 Call_F32, Call_F64, Call_F80,
2203 Call_F128, Call_PPCF128);
2204 ExpandFPLibCall(Node, LC, Results);
2205}
2206
2207SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2208 RTLIB::Libcall Call_I8,
2209 RTLIB::Libcall Call_I16,
2210 RTLIB::Libcall Call_I32,
2211 RTLIB::Libcall Call_I64,
2212 RTLIB::Libcall Call_I128) {
2213 RTLIB::Libcall LC;
2214 switch (Node->getSimpleValueType(0).SimpleTy) {
2215 default: llvm_unreachable("Unexpected request for libcall!");
2216 case MVT::i8: LC = Call_I8; break;
2217 case MVT::i16: LC = Call_I16; break;
2218 case MVT::i32: LC = Call_I32; break;
2219 case MVT::i64: LC = Call_I64; break;
2220 case MVT::i128: LC = Call_I128; break;
2221 }
2222 return ExpandLibCall(LC, Node, isSigned).first;
2223}
2224
2225/// Expand the node to a libcall based on first argument type (for instance
2226/// lround and its variant).
2227void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2228 RTLIB::Libcall Call_F32,
2229 RTLIB::Libcall Call_F64,
2230 RTLIB::Libcall Call_F80,
2231 RTLIB::Libcall Call_F128,
2232 RTLIB::Libcall Call_PPCF128,
2234 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2236 Call_F32, Call_F64, Call_F80,
2237 Call_F128, Call_PPCF128);
2238 ExpandFPLibCall(Node, LC, Results);
2239}
2240
2241/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2242void
2243SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2245 unsigned Opcode = Node->getOpcode();
2246 bool isSigned = Opcode == ISD::SDIVREM;
2247
2248 RTLIB::Libcall LC;
2249 switch (Node->getSimpleValueType(0).SimpleTy) {
2250 default: llvm_unreachable("Unexpected request for libcall!");
2251 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2252 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2253 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2254 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2255 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2256 }
2257
2258 // The input chain to this libcall is the entry node of the function.
2259 // Legalizing the call will automatically add the previous call to the
2260 // dependence.
2261 SDValue InChain = DAG.getEntryNode();
2262
2263 EVT RetVT = Node->getValueType(0);
2264 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2265
2268 for (const SDValue &Op : Node->op_values()) {
2269 EVT ArgVT = Op.getValueType();
2270 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2271 Entry.Node = Op;
2272 Entry.Ty = ArgTy;
2273 Entry.IsSExt = isSigned;
2274 Entry.IsZExt = !isSigned;
2275 Args.push_back(Entry);
2276 }
2277
2278 // Also pass the return address of the remainder.
2279 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2280 Entry.Node = FIPtr;
2281 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2282 Entry.IsSExt = isSigned;
2283 Entry.IsZExt = !isSigned;
2284 Args.push_back(Entry);
2285
2286 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2287 TLI.getPointerTy(DAG.getDataLayout()));
2288
2289 SDLoc dl(Node);
2291 CLI.setDebugLoc(dl)
2292 .setChain(InChain)
2293 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2294 std::move(Args))
2295 .setSExtResult(isSigned)
2296 .setZExtResult(!isSigned);
2297
2298 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2299
2300 // Remainder is loaded back from the stack frame.
2301 SDValue Rem =
2302 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2303 Results.push_back(CallInfo.first);
2304 Results.push_back(Rem);
2305}
2306
2307/// Return true if sincos libcall is available.
2309 RTLIB::Libcall LC;
2310 switch (Node->getSimpleValueType(0).SimpleTy) {
2311 default: llvm_unreachable("Unexpected request for libcall!");
2312 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2313 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2314 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2315 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2316 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2317 }
2318 return TLI.getLibcallName(LC) != nullptr;
2319}
2320
2321/// Only issue sincos libcall if both sin and cos are needed.
2322static bool useSinCos(SDNode *Node) {
2323 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2324 ? ISD::FCOS : ISD::FSIN;
2325
2326 SDValue Op0 = Node->getOperand(0);
2327 for (const SDNode *User : Op0.getNode()->uses()) {
2328 if (User == Node)
2329 continue;
2330 // The other user might have been turned into sincos already.
2331 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2332 return true;
2333 }
2334 return false;
2335}
2336
2337/// Issue libcalls to sincos to compute sin / cos pairs.
2338void
2339SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2341 RTLIB::Libcall LC;
2342 switch (Node->getSimpleValueType(0).SimpleTy) {
2343 default: llvm_unreachable("Unexpected request for libcall!");
2344 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2345 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2346 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2347 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2348 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2349 }
2350
2351 // The input chain to this libcall is the entry node of the function.
2352 // Legalizing the call will automatically add the previous call to the
2353 // dependence.
2354 SDValue InChain = DAG.getEntryNode();
2355
2356 EVT RetVT = Node->getValueType(0);
2357 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2358
2361
2362 // Pass the argument.
2363 Entry.Node = Node->getOperand(0);
2364 Entry.Ty = RetTy;
2365 Entry.IsSExt = false;
2366 Entry.IsZExt = false;
2367 Args.push_back(Entry);
2368
2369 // Pass the return address of sin.
2370 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2371 Entry.Node = SinPtr;
2372 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2373 Entry.IsSExt = false;
2374 Entry.IsZExt = false;
2375 Args.push_back(Entry);
2376
2377 // Also pass the return address of the cos.
2378 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2379 Entry.Node = CosPtr;
2380 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2381 Entry.IsSExt = false;
2382 Entry.IsZExt = false;
2383 Args.push_back(Entry);
2384
2385 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2386 TLI.getPointerTy(DAG.getDataLayout()));
2387
2388 SDLoc dl(Node);
2390 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2391 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2392 std::move(Args));
2393
2394 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2395
2396 Results.push_back(
2397 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2398 Results.push_back(
2399 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2400}
2401
2402SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2403 SDLoc dl(Node);
2404 EVT VT = Node->getValueType(0);
2405 SDValue X = Node->getOperand(0);
2406 SDValue N = Node->getOperand(1);
2407 EVT ExpVT = N.getValueType();
2408 EVT AsIntVT = VT.changeTypeToInteger();
2409 if (AsIntVT == EVT()) // TODO: How to handle f80?
2410 return SDValue();
2411
2412 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2413 return SDValue();
2414
2415 SDNodeFlags NSW;
2416 NSW.setNoSignedWrap(true);
2417 SDNodeFlags NUW_NSW;
2418 NUW_NSW.setNoUnsignedWrap(true);
2419 NUW_NSW.setNoSignedWrap(true);
2420
2421 EVT SetCCVT =
2422 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2424
2425 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2426 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2427 const int Precision = APFloat::semanticsPrecision(FltSem);
2428
2429 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2430 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2431
2432 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2433
2434 const APFloat One(FltSem, "1.0");
2435 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2436
2437 // Offset by precision to avoid denormal range.
2438 APFloat ScaleDownK =
2439 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2440
2441 // TODO: Should really introduce control flow and use a block for the >
2442 // MaxExp, < MinExp cases
2443
2444 // First, handle exponents Exp > MaxExp and scale down.
2445 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2446
2447 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2448 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2449 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2450 SDValue DecN1 =
2451 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2452
2453 SDValue ScaleUpTwice =
2454 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2455
2456 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2457 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2458 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2459
2460 SDValue SelectN_Big =
2461 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2462 SDValue SelectX_Big =
2463 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2464
2465 // Now handle exponents Exp < MinExp
2466 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2467
2468 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2469 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2470
2471 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2472
2473 SDValue ClampMinVal =
2474 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2475 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2476 SDValue IncN1 =
2477 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2478
2479 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2480 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2481 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2482
2483 SDValue ScaleDownTwice = DAG.getSetCC(
2484 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2485 ISD::SETULT);
2486
2487 SDValue SelectN_Small =
2488 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2489 SDValue SelectX_Small =
2490 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2491
2492 // Now combine the two out of range exponent handling cases with the base
2493 // case.
2494 SDValue NewX = DAG.getNode(
2495 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2496 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2497
2498 SDValue NewN = DAG.getNode(
2499 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2500 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2501
2502 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2503
2504 SDValue ExponentShiftAmt =
2505 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2506 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2507
2508 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2509 ExponentShiftAmt, NUW_NSW);
2510 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2511 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2512}
2513
2514SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2515 SDLoc dl(Node);
2516 SDValue Val = Node->getOperand(0);
2517 EVT VT = Val.getValueType();
2518 EVT ExpVT = Node->getValueType(1);
2519 EVT AsIntVT = VT.changeTypeToInteger();
2520 if (AsIntVT == EVT()) // TODO: How to handle f80?
2521 return SDValue();
2522
2524 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2525 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2526 const unsigned BitSize = VT.getScalarSizeInBits();
2527
2528 // TODO: Could introduce control flow and skip over the denormal handling.
2529
2530 // scale_up = fmul value, scalbn(1.0, precision + 1)
2531 // extracted_exp = (bitcast value to uint) >> precision - 1
2532 // biased_exp = extracted_exp + min_exp
2533 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2534 //
2535 // is_denormal = val < smallest_normalized
2536 // computed_fract = is_denormal ? scale_up : extracted_fract
2537 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2538 //
2539 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2540 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2541
2542 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2543 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2544 AsIntVT);
2545
2546 SDValue SmallestNormalizedInt = DAG.getConstant(
2547 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2548 AsIntVT);
2549
2550 // Masks out the exponent bits.
2551 SDValue ExpMask =
2552 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2553
2554 // Mask out the exponent part of the value.
2555 //
2556 // e.g, for f32 FractSignMaskVal = 0x807fffff
2557 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2558 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2559
2560 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2561 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2562
2563 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2564
2565 const APFloat One(FltSem, "1.0");
2566 // Scale a possible denormal input.
2567 // e.g., for f64, 0x1p+54
2568 APFloat ScaleUpKVal =
2569 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2570
2571 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2572 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2573
2574 EVT SetCCVT =
2575 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2576
2577 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2578
2579 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2580
2581 SDValue AddNegSmallestNormal =
2582 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2583 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2584 NegSmallestNormalizedInt, ISD::SETULE);
2585
2586 SDValue IsDenormal =
2587 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2588
2589 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2590 SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2591
2592 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2593 SDValue ScaledSelect =
2594 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2595
2596 SDValue ExpMaskScaled =
2597 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2598
2599 SDValue ScaledValue =
2600 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2601
2602 // Extract the exponent bits.
2603 SDValue ExponentShiftAmt =
2604 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2605 SDValue ShiftedExp =
2606 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2607 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2608
2609 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2610 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2611 SDValue DenormalExpBias =
2612 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2613
2614 SDValue MaskedFractAsInt =
2615 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2616 const APFloat Half(FltSem, "0.5");
2617 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2618 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2619 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2620
2621 SDValue ComputedExp =
2622 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2623
2624 SDValue Result0 =
2625 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2626
2627 SDValue Result1 =
2628 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2629
2630 return DAG.getMergeValues({Result0, Result1}, dl);
2631}
2632
2633/// This function is responsible for legalizing a
2634/// INT_TO_FP operation of the specified operand when the target requests that
2635/// we expand it. At this point, we know that the result and operand types are
2636/// legal for the target.
2637SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2638 SDValue &Chain) {
2639 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2640 Node->getOpcode() == ISD::SINT_TO_FP);
2641 EVT DestVT = Node->getValueType(0);
2642 SDLoc dl(Node);
2643 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2644 SDValue Op0 = Node->getOperand(OpNo);
2645 EVT SrcVT = Op0.getValueType();
2646
2647 // TODO: Should any fast-math-flags be set for the created nodes?
2648 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2649 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2650 (DestVT.bitsLE(MVT::f64) ||
2651 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2653 DestVT))) {
2654 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2655 "expansion\n");
2656
2657 // Get the stack frame index of a 8 byte buffer.
2658 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2659
2660 SDValue Lo = Op0;
2661 // if signed map to unsigned space
2662 if (isSigned) {
2663 // Invert sign bit (signed to unsigned mapping).
2664 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2665 DAG.getConstant(0x80000000u, dl, MVT::i32));
2666 }
2667 // Initial hi portion of constructed double.
2668 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2669
2670 // If this a big endian target, swap the lo and high data.
2671 if (DAG.getDataLayout().isBigEndian())
2672 std::swap(Lo, Hi);
2673
2674 SDValue MemChain = DAG.getEntryNode();
2675
2676 // Store the lo of the constructed double.
2677 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2679 // Store the hi of the constructed double.
2680 SDValue HiPtr =
2681 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2682 SDValue Store2 =
2683 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2684 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2685
2686 // load the constructed double
2687 SDValue Load =
2688 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2689 // FP constant to bias correct the final result
2690 SDValue Bias = DAG.getConstantFP(
2691 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2692 : llvm::bit_cast<double>(0x4330000000000000ULL),
2693 dl, MVT::f64);
2694 // Subtract the bias and get the final result.
2695 SDValue Sub;
2697 if (Node->isStrictFPOpcode()) {
2698 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2699 {Node->getOperand(0), Load, Bias});
2700 Chain = Sub.getValue(1);
2701 if (DestVT != Sub.getValueType()) {
2702 std::pair<SDValue, SDValue> ResultPair;
2703 ResultPair =
2704 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2705 Result = ResultPair.first;
2706 Chain = ResultPair.second;
2707 }
2708 else
2709 Result = Sub;
2710 } else {
2711 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2712 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2713 }
2714 return Result;
2715 }
2716
2717 if (isSigned)
2718 return SDValue();
2719
2720 // TODO: Generalize this for use with other types.
2721 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2722 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2723 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2724 // For unsigned conversions, convert them to signed conversions using the
2725 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2726 // should be valid for i32->f32 as well.
2727
2728 // More generally this transform should be valid if there are 3 more bits
2729 // in the integer type than the significand. Rounding uses the first bit
2730 // after the width of the significand and the OR of all bits after that. So
2731 // we need to be able to OR the shifted out bit into one of the bits that
2732 // participate in the OR.
2733
2734 // TODO: This really should be implemented using a branch rather than a
2735 // select. We happen to get lucky and machinesink does the right
2736 // thing most of the time. This would be a good candidate for a
2737 // pseudo-op, or, even better, for whole-function isel.
2738 EVT SetCCVT = getSetCCResultType(SrcVT);
2739
2740 SDValue SignBitTest = DAG.getSetCC(
2741 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2742
2743 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2744 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2745 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2746 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2747 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2748 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2749
2750 SDValue Slow, Fast;
2751 if (Node->isStrictFPOpcode()) {
2752 // In strict mode, we must avoid spurious exceptions, and therefore
2753 // must make sure to only emit a single STRICT_SINT_TO_FP.
2754 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2755 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2756 { Node->getOperand(0), InCvt });
2757 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2758 { Fast.getValue(1), Fast, Fast });
2759 Chain = Slow.getValue(1);
2760 // The STRICT_SINT_TO_FP inherits the exception mode from the
2761 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2762 // never raise any exception.
2764 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2765 Fast->setFlags(Flags);
2766 Flags.setNoFPExcept(true);
2767 Slow->setFlags(Flags);
2768 } else {
2769 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2770 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2771 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2772 }
2773
2774 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2775 }
2776
2777 // Don't expand it if there isn't cheap fadd.
2778 if (!TLI.isOperationLegalOrCustom(
2779 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2780 return SDValue();
2781
2782 // The following optimization is valid only if every value in SrcVT (when
2783 // treated as signed) is representable in DestVT. Check that the mantissa
2784 // size of DestVT is >= than the number of bits in SrcVT -1.
2785 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2786 SrcVT.getSizeInBits() - 1 &&
2787 "Cannot perform lossless SINT_TO_FP!");
2788
2789 SDValue Tmp1;
2790 if (Node->isStrictFPOpcode()) {
2791 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2792 { Node->getOperand(0), Op0 });
2793 } else
2794 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2795
2796 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2797 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2798 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2799 Four = DAG.getIntPtrConstant(4, dl);
2800 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2801 SignSet, Four, Zero);
2802
2803 // If the sign bit of the integer is set, the large number will be treated
2804 // as a negative number. To counteract this, the dynamic code adds an
2805 // offset depending on the data type.
2806 uint64_t FF;
2807 switch (SrcVT.getSimpleVT().SimpleTy) {
2808 default:
2809 return SDValue();
2810 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2811 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2812 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2813 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2814 }
2815 if (DAG.getDataLayout().isLittleEndian())
2816 FF <<= 32;
2817 Constant *FudgeFactor = ConstantInt::get(
2818 Type::getInt64Ty(*DAG.getContext()), FF);
2819
2820 SDValue CPIdx =
2821 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2822 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2823 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2824 Alignment = commonAlignment(Alignment, 4);
2825 SDValue FudgeInReg;
2826 if (DestVT == MVT::f32)
2827 FudgeInReg = DAG.getLoad(
2828 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2829 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2830 Alignment);
2831 else {
2832 SDValue Load = DAG.getExtLoad(
2833 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2834 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2835 Alignment);
2836 HandleSDNode Handle(Load);
2837 LegalizeOp(Load.getNode());
2838 FudgeInReg = Handle.getValue();
2839 }
2840
2841 if (Node->isStrictFPOpcode()) {
2842 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2843 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2844 Chain = Result.getValue(1);
2845 return Result;
2846 }
2847
2848 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2849}
2850
2851/// This function is responsible for legalizing a
2852/// *INT_TO_FP operation of the specified operand when the target requests that
2853/// we promote it. At this point, we know that the result and operand types are
2854/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2855/// operation that takes a larger input.
2856void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2858 bool IsStrict = N->isStrictFPOpcode();
2859 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2860 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2861 EVT DestVT = N->getValueType(0);
2862 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2863 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2864 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2865
2866 // First step, figure out the appropriate *INT_TO_FP operation to use.
2867 EVT NewInTy = LegalOp.getValueType();
2868
2869 unsigned OpToUse = 0;
2870
2871 // Scan for the appropriate larger type to use.
2872 while (true) {
2873 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2874 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2875
2876 // If the target supports SINT_TO_FP of this type, use it.
2877 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2878 OpToUse = SIntOp;
2879 break;
2880 }
2881 if (IsSigned)
2882 continue;
2883
2884 // If the target supports UINT_TO_FP of this type, use it.
2885 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2886 OpToUse = UIntOp;
2887 break;
2888 }
2889
2890 // Otherwise, try a larger type.
2891 }
2892
2893 // Okay, we found the operation and type to use. Zero extend our input to the
2894 // desired type then run the operation on it.
2895 if (IsStrict) {
2896 SDValue Res =
2897 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2898 {N->getOperand(0),
2899 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2900 dl, NewInTy, LegalOp)});
2901 Results.push_back(Res);
2902 Results.push_back(Res.getValue(1));
2903 return;
2904 }
2905
2906 Results.push_back(
2907 DAG.getNode(OpToUse, dl, DestVT,
2908 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2909 dl, NewInTy, LegalOp)));
2910}
2911
2912/// This function is responsible for legalizing a
2913/// FP_TO_*INT operation of the specified operand when the target requests that
2914/// we promote it. At this point, we know that the result and operand types are
2915/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2916/// operation that returns a larger result.
2917void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2919 bool IsStrict = N->isStrictFPOpcode();
2920 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2921 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2922 EVT DestVT = N->getValueType(0);
2923 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2924 // First step, figure out the appropriate FP_TO*INT operation to use.
2925 EVT NewOutTy = DestVT;
2926
2927 unsigned OpToUse = 0;
2928
2929 // Scan for the appropriate larger type to use.
2930 while (true) {
2931 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2932 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2933
2934 // A larger signed type can hold all unsigned values of the requested type,
2935 // so using FP_TO_SINT is valid
2936 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2937 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2938 break;
2939
2940 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2941 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2942 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2943 break;
2944
2945 // Otherwise, try a larger type.
2946 }
2947
2948 // Okay, we found the operation and type to use.
2950 if (IsStrict) {
2951 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2952 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2953 } else
2954 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2955
2956 // Truncate the result of the extended FP_TO_*INT operation to the desired
2957 // size.
2958 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2959 Results.push_back(Trunc);
2960 if (IsStrict)
2961 Results.push_back(Operation.getValue(1));
2962}
2963
2964/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2965/// the result and operand types are legal and there must be a legal
2966/// FP_TO_*INT_SAT operation for a larger result type.
2967SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2968 const SDLoc &dl) {
2969 unsigned Opcode = Node->getOpcode();
2970
2971 // Scan for the appropriate larger type to use.
2972 EVT NewOutTy = Node->getValueType(0);
2973 while (true) {
2974 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2975 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2976
2977 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2978 break;
2979 }
2980
2981 // Saturation width is determined by second operand, so we don't have to
2982 // perform any fixup and can directly truncate the result.
2983 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2984 Node->getOperand(1));
2985 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2986}
2987
2988/// Open code the operations for PARITY of the specified operation.
2989SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2990 EVT VT = Op.getValueType();
2991 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2992 unsigned Sz = VT.getScalarSizeInBits();
2993
2994 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2996 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2997 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2998 } else {
2999 Result = Op;
3000 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
3001 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
3002 DAG.getConstant(1ULL << (--i), dl, ShVT));
3003 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
3004 }
3005 }
3006
3007 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3008}
3009
3010bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3011 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3013 SDLoc dl(Node);
3014 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3015 bool NeedInvert;
3016 switch (Node->getOpcode()) {
3017 case ISD::ABS:
3018 if ((Tmp1 = TLI.expandABS(Node, DAG)))
3019 Results.push_back(Tmp1);
3020 break;
3021 case ISD::ABDS:
3022 case ISD::ABDU:
3023 if ((Tmp1 = TLI.expandABD(Node, DAG)))
3024 Results.push_back(Tmp1);
3025 break;
3026 case ISD::CTPOP:
3027 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3028 Results.push_back(Tmp1);
3029 break;
3030 case ISD::CTLZ:
3032 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3033 Results.push_back(Tmp1);
3034 break;
3035 case ISD::CTTZ:
3037 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3038 Results.push_back(Tmp1);
3039 break;
3040 case ISD::BITREVERSE:
3041 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3042 Results.push_back(Tmp1);
3043 break;
3044 case ISD::BSWAP:
3045 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3046 Results.push_back(Tmp1);
3047 break;
3048 case ISD::PARITY:
3049 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3050 break;
3051 case ISD::FRAMEADDR:
3052 case ISD::RETURNADDR:
3054 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3055 break;
3056 case ISD::EH_DWARF_CFA: {
3057 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3058 TLI.getPointerTy(DAG.getDataLayout()));
3059 SDValue Offset = DAG.getNode(ISD::ADD, dl,
3060 CfaArg.getValueType(),
3061 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3062 CfaArg.getValueType()),
3063 CfaArg);
3064 SDValue FA = DAG.getNode(
3065 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3066 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3067 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3068 FA, Offset));
3069 break;
3070 }
3071 case ISD::GET_ROUNDING:
3072 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3073 Results.push_back(Node->getOperand(0));
3074 break;
3075 case ISD::EH_RETURN:
3076 case ISD::EH_LABEL:
3077 case ISD::PREFETCH:
3078 case ISD::VAEND:
3080 // If the target didn't expand these, there's nothing to do, so just
3081 // preserve the chain and be done.
3082 Results.push_back(Node->getOperand(0));
3083 break;
3086 // If the target didn't expand this, just return 'zero' and preserve the
3087 // chain.
3088 Results.append(Node->getNumValues() - 1,
3089 DAG.getConstant(0, dl, Node->getValueType(0)));
3090 Results.push_back(Node->getOperand(0));
3091 break;
3093 // If the target didn't expand this, just return 'zero' and preserve the
3094 // chain.
3095 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3096 Results.push_back(Node->getOperand(0));
3097 break;
3098 case ISD::ATOMIC_LOAD: {
3099 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3100 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3101 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3102 SDValue Swap = DAG.getAtomicCmpSwap(
3103 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3104 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3105 cast<AtomicSDNode>(Node)->getMemOperand());
3106 Results.push_back(Swap.getValue(0));
3107 Results.push_back(Swap.getValue(1));
3108 break;
3109 }
3110 case ISD::ATOMIC_STORE: {
3111 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3112 SDValue Swap = DAG.getAtomic(
3113 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3114 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3115 cast<AtomicSDNode>(Node)->getMemOperand());
3116 Results.push_back(Swap.getValue(1));
3117 break;
3118 }
3120 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3121 // splits out the success value as a comparison. Expanding the resulting
3122 // ATOMIC_CMP_SWAP will produce a libcall.
3123 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3124 SDValue Res = DAG.getAtomicCmpSwap(
3125 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3126 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3127 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3128
3129 SDValue ExtRes = Res;
3130 SDValue LHS = Res;
3131 SDValue RHS = Node->getOperand(1);
3132
3133 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3134 EVT OuterType = Node->getValueType(0);
3135 switch (TLI.getExtendForAtomicOps()) {
3136 case ISD::SIGN_EXTEND:
3137 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3138 DAG.getValueType(AtomicType));
3139 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3140 Node->getOperand(2), DAG.getValueType(AtomicType));
3141 ExtRes = LHS;
3142 break;
3143 case ISD::ZERO_EXTEND:
3144 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3145 DAG.getValueType(AtomicType));
3146 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3147 ExtRes = LHS;
3148 break;
3149 case ISD::ANY_EXTEND:
3150 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3151 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3152 break;
3153 default:
3154 llvm_unreachable("Invalid atomic op extension");
3155 }
3156
3158 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3159
3160 Results.push_back(ExtRes.getValue(0));
3161 Results.push_back(Success);
3162 Results.push_back(Res.getValue(1));
3163 break;
3164 }
3165 case ISD::ATOMIC_LOAD_SUB: {
3166 SDLoc DL(Node);
3167 EVT VT = Node->getValueType(0);
3168 SDValue RHS = Node->getOperand(2);
3169 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3170 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3171 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3172 RHS = RHS->getOperand(0);
3173 SDValue NewRHS =
3174 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3175 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3176 Node->getOperand(0), Node->getOperand(1),
3177 NewRHS, AN->getMemOperand());
3178 Results.push_back(Res);
3179 Results.push_back(Res.getValue(1));
3180 break;
3181 }
3183 ExpandDYNAMIC_STACKALLOC(Node, Results);
3184 break;
3185 case ISD::MERGE_VALUES:
3186 for (unsigned i = 0; i < Node->getNumValues(); i++)
3187 Results.push_back(Node->getOperand(i));
3188 break;
3189 case ISD::UNDEF: {
3190 EVT VT = Node->getValueType(0);
3191 if (VT.isInteger())
3192 Results.push_back(DAG.getConstant(0, dl, VT));
3193 else {
3194 assert(VT.isFloatingPoint() && "Unknown value type!");
3195 Results.push_back(DAG.getConstantFP(0, dl, VT));
3196 }
3197 break;
3198 }
3200 // When strict mode is enforced we can't do expansion because it
3201 // does not honor the "strict" properties. Only libcall is allowed.
3202 if (TLI.isStrictFPEnabled())
3203 break;
3204 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3205 // since this operation is more efficient than stack operation.
3206 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3207 Node->getValueType(0))
3208 == TargetLowering::Legal)
3209 break;
3210 // We fall back to use stack operation when the FP_ROUND operation
3211 // isn't available.
3212 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3213 Node->getValueType(0), dl,
3214 Node->getOperand(0)))) {
3215 ReplaceNode(Node, Tmp1.getNode());
3216 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3217 return true;
3218 }
3219 break;
3220 case ISD::FP_ROUND: {
3221 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3222 Results.push_back(Tmp1);
3223 break;
3224 }
3225
3227 }
3228 case ISD::BITCAST:
3229 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3230 Node->getValueType(0), dl)))
3231 Results.push_back(Tmp1);
3232 break;
3234 // When strict mode is enforced we can't do expansion because it
3235 // does not honor the "strict" properties. Only libcall is allowed.
3236 if (TLI.isStrictFPEnabled())
3237 break;
3238 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3239 // since this operation is more efficient than stack operation.
3240 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3241 Node->getValueType(0))
3242 == TargetLowering::Legal)
3243 break;
3244 // We fall back to use stack operation when the FP_EXTEND operation
3245 // isn't available.
3246 if ((Tmp1 = EmitStackConvert(
3247 Node->getOperand(1), Node->getOperand(1).getValueType(),
3248 Node->getValueType(0), dl, Node->getOperand(0)))) {
3249 ReplaceNode(Node, Tmp1.getNode());
3250 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3251 return true;
3252 }
3253 break;
3254 case ISD::FP_EXTEND: {
3255 SDValue Op = Node->getOperand(0);
3256 EVT SrcVT = Op.getValueType();
3257 EVT DstVT = Node->getValueType(0);
3258 if (SrcVT.getScalarType() == MVT::bf16) {
3259 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3260 break;
3261 }
3262
3263 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3264 Results.push_back(Tmp1);
3265 break;
3266 }
3267 case ISD::BF16_TO_FP: {
3268 // Always expand bf16 to f32 casts, they lower to ext + shift.
3269 //
3270 // Note that the operand of this code can be bf16 or an integer type in case
3271 // bf16 is not supported on the target and was softened.
3272 SDValue Op = Node->getOperand(0);
3273 if (Op.getValueType() == MVT::bf16) {
3274 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3275 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3276 } else {
3277 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3278 }
3279 Op = DAG.getNode(
3280 ISD::SHL, dl, MVT::i32, Op,
3281 DAG.getConstant(16, dl,
3282 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3283 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3284 // Add fp_extend in case the output is bigger than f32.
3285 if (Node->getValueType(0) != MVT::f32)
3286 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3287 Results.push_back(Op);
3288 break;
3289 }
3290 case ISD::FP_TO_BF16: {
3291 SDValue Op = Node->getOperand(0);
3292 if (Op.getValueType() != MVT::f32)
3293 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3294 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3295 // Certain SNaNs will turn into infinities if we do a simple shift right.
3296 if (!DAG.isKnownNeverSNaN(Op)) {
3297 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());
3298 }
3299 Op = DAG.getNode(
3300 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3301 DAG.getConstant(16, dl,
3302 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3303 // The result of this node can be bf16 or an integer type in case bf16 is
3304 // not supported on the target and was softened to i16 for storage.
3305 if (Node->getValueType(0) == MVT::bf16) {
3306 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3307 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3308 } else {
3309 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3310 }
3311 Results.push_back(Op);
3312 break;
3313 }
3315 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3316 EVT VT = Node->getValueType(0);
3317
3318 // An in-register sign-extend of a boolean is a negation:
3319 // 'true' (1) sign-extended is -1.
3320 // 'false' (0) sign-extended is 0.
3321 // However, we must mask the high bits of the source operand because the
3322 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3323
3324 // TODO: Do this for vectors too?
3325 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3326 SDValue One = DAG.getConstant(1, dl, VT);
3327 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3328 SDValue Zero = DAG.getConstant(0, dl, VT);
3329 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3330 Results.push_back(Neg);
3331 break;
3332 }
3333
3334 // NOTE: we could fall back on load/store here too for targets without
3335 // SRA. However, it is doubtful that any exist.
3336 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3337 unsigned BitsDiff = VT.getScalarSizeInBits() -
3338 ExtraVT.getScalarSizeInBits();
3339 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3340 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3341 Node->getOperand(0), ShiftCst);
3342 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3343 Results.push_back(Tmp1);
3344 break;
3345 }
3346 case ISD::UINT_TO_FP:
3348 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3349 Results.push_back(Tmp1);
3350 if (Node->isStrictFPOpcode())
3351 Results.push_back(Tmp2);
3352 break;
3353 }
3354 [[fallthrough]];
3355 case ISD::SINT_TO_FP:
3357 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3358 Results.push_back(Tmp1);
3359 if (Node->isStrictFPOpcode())
3360 Results.push_back(Tmp2);
3361 }
3362 break;
3363 case ISD::FP_TO_SINT:
3364 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3365 Results.push_back(Tmp1);
3366 break;
3368 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3369 ReplaceNode(Node, Tmp1.getNode());
3370 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3371 return true;
3372 }
3373 break;
3374 case ISD::FP_TO_UINT:
3375 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3376 Results.push_back(Tmp1);
3377 break;
3379 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3380 // Relink the chain.
3381 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3382 // Replace the new UINT result.
3383 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3384 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3385 return true;
3386 }
3387 break;
3390 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3391 break;
3392 case ISD::VAARG:
3393 Results.push_back(DAG.expandVAArg(Node));
3394 Results.push_back(Results[0].getValue(1));
3395 break;
3396 case ISD::VACOPY:
3397 Results.push_back(DAG.expandVACopy(Node));
3398 break;
3400 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3401 // This must be an access of the only element. Return it.
3402 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3403 Node->getOperand(0));
3404 else
3405 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3406 Results.push_back(Tmp1);
3407 break;
3409 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3410 break;
3412 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3413 break;
3415 Results.push_back(ExpandVectorBuildThroughStack(Node));
3416 break;
3418 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3419 break;
3421 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3422 Node->getOperand(1),
3423 Node->getOperand(2), dl));
3424 break;
3425 case ISD::VECTOR_SHUFFLE: {
3426 SmallVector<int, 32> NewMask;
3427 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3428
3429 EVT VT = Node->getValueType(0);
3430 EVT EltVT = VT.getVectorElementType();
3431 SDValue Op0 = Node->getOperand(0);
3432 SDValue Op1 = Node->getOperand(1);
3433 if (!TLI.isTypeLegal(EltVT)) {
3434 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3435
3436 // BUILD_VECTOR operands are allowed to be wider than the element type.
3437 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3438 // it.
3439 if (NewEltVT.bitsLT(EltVT)) {
3440 // Convert shuffle node.
3441 // If original node was v4i64 and the new EltVT is i32,
3442 // cast operands to v8i32 and re-build the mask.
3443
3444 // Calculate new VT, the size of the new VT should be equal to original.
3445 EVT NewVT =
3446 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3447 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3448 assert(NewVT.bitsEq(VT));
3449
3450 // cast operands to new VT
3451 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3452 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3453
3454 // Convert the shuffle mask
3455 unsigned int factor =
3457
3458 // EltVT gets smaller
3459 assert(factor > 0);
3460
3461 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3462 if (Mask[i] < 0) {
3463 for (unsigned fi = 0; fi < factor; ++fi)
3464 NewMask.push_back(Mask[i]);
3465 }
3466 else {
3467 for (unsigned fi = 0; fi < factor; ++fi)
3468 NewMask.push_back(Mask[i]*factor+fi);
3469 }
3470 }
3471 Mask = NewMask;
3472 VT = NewVT;
3473 }
3474 EltVT = NewEltVT;
3475 }
3476 unsigned NumElems = VT.getVectorNumElements();
3478 for (unsigned i = 0; i != NumElems; ++i) {
3479 if (Mask[i] < 0) {
3480 Ops.push_back(DAG.getUNDEF(EltVT));
3481 continue;
3482 }
3483 unsigned Idx = Mask[i];
3484 if (Idx < NumElems)
3485 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3486 DAG.getVectorIdxConstant(Idx, dl)));
3487 else
3488 Ops.push_back(
3489 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3490 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3491 }
3492
3493 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3494 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3495 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3496 Results.push_back(Tmp1);
3497 break;
3498 }
3499 case ISD::VECTOR_SPLICE: {
3500 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3501 break;
3502 }
3503 case ISD::EXTRACT_ELEMENT: {
3504 EVT OpTy = Node->getOperand(0).getValueType();
3505 if (Node->getConstantOperandVal(1)) {
3506 // 1 -> Hi
3507 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3508 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3509 TLI.getShiftAmountTy(
3510 Node->getOperand(0).getValueType(),
3511 DAG.getDataLayout())));
3512 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3513 } else {
3514 // 0 -> Lo
3515 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3516 Node->getOperand(0));
3517 }
3518 Results.push_back(Tmp1);
3519 break;
3520 }
3521 case ISD::STACKSAVE:
3522 // Expand to CopyFromReg if the target set
3523 // StackPointerRegisterToSaveRestore.
3524 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3525 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3526 Node->getValueType(0)));
3527 Results.push_back(Results[0].getValue(1));
3528 } else {
3529 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3530 Results.push_back(Node->getOperand(0));
3531 }
3532 break;
3533 case ISD::STACKRESTORE:
3534 // Expand to CopyToReg if the target set
3535 // StackPointerRegisterToSaveRestore.
3536 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3537 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3538 Node->getOperand(1)));
3539 } else {
3540 Results.push_back(Node->getOperand(0));
3541 }
3542 break;
3544 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3545 Results.push_back(Results[0].getValue(0));
3546 break;
3547 case ISD::FCOPYSIGN:
3548 Results.push_back(ExpandFCOPYSIGN(Node));
3549 break;
3550 case ISD::FNEG:
3551 Results.push_back(ExpandFNEG(Node));
3552 break;
3553 case ISD::FABS:
3554 Results.push_back(ExpandFABS(Node));
3555 break;
3556 case ISD::IS_FPCLASS: {
3557 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));
3558 if (SDValue Expanded =
3559 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3560 Test, Node->getFlags(), SDLoc(Node), DAG))
3561 Results.push_back(Expanded);
3562 break;
3563 }
3564 case ISD::SMIN:
3565 case ISD::SMAX:
3566 case ISD::UMIN:
3567 case ISD::UMAX: {
3568 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3569 ISD::CondCode Pred;
3570 switch (Node->getOpcode()) {
3571 default: llvm_unreachable("How did we get here?");
3572 case ISD::SMAX: Pred = ISD::SETGT; break;
3573 case ISD::SMIN: Pred = ISD::SETLT; break;
3574 case ISD::UMAX: Pred = ISD::SETUGT; break;
3575 case ISD::UMIN: Pred = ISD::SETULT; break;
3576 }
3577 Tmp1 = Node->getOperand(0);
3578 Tmp2 = Node->getOperand(1);
3579 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3580 Results.push_back(Tmp1);
3581 break;
3582 }
3583 case ISD::FMINNUM:
3584 case ISD::FMAXNUM: {
3585 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3586 Results.push_back(Expanded);
3587 break;
3588 }
3589 case ISD::FSIN:
3590 case ISD::FCOS: {
3591 EVT VT = Node->getValueType(0);
3592 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3593 // fcos which share the same operand and both are used.
3594 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3596 && useSinCos(Node)) {
3597 SDVTList VTs = DAG.getVTList(VT, VT);
3598 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3599 if (Node->getOpcode() == ISD::FCOS)
3600 Tmp1 = Tmp1.getValue(1);
3601 Results.push_back(Tmp1);
3602 }
3603 break;
3604 }
3605 case ISD::FLDEXP:
3606 case ISD::STRICT_FLDEXP: {
3607 EVT VT = Node->getValueType(0);
3609 // Use the LibCall instead, it is very likely faster
3610 // FIXME: Use separate LibCall action.
3611 if (TLI.getLibcallName(LC))
3612 break;
3613
3614 if (SDValue Expanded = expandLdexp(Node)) {
3615 Results.push_back(Expanded);
3616 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3617 Results.push_back(Expanded.getValue(1));
3618 }
3619
3620 break;
3621 }
3622 case ISD::FFREXP: {
3623 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3624 // Use the LibCall instead, it is very likely faster
3625 // FIXME: Use separate LibCall action.
3626 if (TLI.getLibcallName(LC))
3627 break;
3628
3629 if (SDValue Expanded = expandFrexp(Node)) {
3630 Results.push_back(Expanded);
3631 Results.push_back(Expanded.getValue(1));
3632 }
3633 break;
3634 }
3635 case ISD::FMAD:
3636 llvm_unreachable("Illegal fmad should never be formed");
3637
3638 case ISD::FP16_TO_FP:
3639 if (Node->getValueType(0) != MVT::f32) {
3640 // We can extend to types bigger than f32 in two steps without changing
3641 // the result. Since "f16 -> f32" is much more commonly available, give
3642 // CodeGen the option of emitting that before resorting to a libcall.
3643 SDValue Res =
3644 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3645 Results.push_back(
3646 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3647 }
3648 break;
3651 if (Node->getValueType(0) != MVT::f32) {
3652 // We can extend to types bigger than f32 in two steps without changing
3653 // the result. Since "f16 -> f32" is much more commonly available, give
3654 // CodeGen the option of emitting that before resorting to a libcall.
3655 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other},
3656 {Node->getOperand(0), Node->getOperand(1)});
3657 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3658 {Node->getValueType(0), MVT::Other},
3659 {Res.getValue(1), Res});
3660 Results.push_back(Res);
3661 Results.push_back(Res.getValue(1));
3662 }
3663 break;
3664 case ISD::FP_TO_FP16:
3665 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3666 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3667 SDValue Op = Node->getOperand(0);
3668 MVT SVT = Op.getSimpleValueType();
3669 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3670 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3671 // Under fastmath, we can expand this node into a fround followed by
3672 // a float-half conversion.
3673 SDValue FloatVal =
3674 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3675 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3676 Results.push_back(
3677 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3678 }
3679 }
3680 break;
3681 case ISD::ConstantFP: {
3682 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3683 // Check to see if this FP immediate is already legal.
3684 // If this is a legal constant, turn it into a TargetConstantFP node.
3685 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3686 DAG.shouldOptForSize()))
3687 Results.push_back(ExpandConstantFP(CFP, true));
3688 break;
3689 }
3690 case ISD::Constant: {
3691 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3692 Results.push_back(ExpandConstant(CP));
3693 break;
3694 }
3695 case ISD::FSUB: {
3696 EVT VT = Node->getValueType(0);
3697 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3698 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3699 const SDNodeFlags Flags = Node->getFlags();
3700 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3701 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3702 Results.push_back(Tmp1);
3703 }
3704 break;
3705 }
3706 case ISD::SUB: {
3707 EVT VT = Node->getValueType(0);
3708 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3709 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3710 "Don't know how to expand this subtraction!");
3711 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3712 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3713 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3714 break;
3715 }
3716 case ISD::UREM:
3717 case ISD::SREM:
3718 if (TLI.expandREM(Node, Tmp1, DAG))
3719 Results.push_back(Tmp1);
3720 break;
3721 case ISD::UDIV:
3722 case ISD::SDIV: {
3723 bool isSigned = Node->getOpcode() == ISD::SDIV;
3724 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3725 EVT VT = Node->getValueType(0);
3726 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3727 SDVTList VTs = DAG.getVTList(VT, VT);
3728 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3729 Node->getOperand(1));
3730 Results.push_back(Tmp1);
3731 }
3732 break;
3733 }
3734 case ISD::MULHU:
3735 case ISD::MULHS: {
3736 unsigned ExpandOpcode =
3737 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3738 EVT VT = Node->getValueType(0);
3739 SDVTList VTs = DAG.getVTList(VT, VT);
3740
3741 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3742 Node->getOperand(1));
3743 Results.push_back(Tmp1.getValue(1));
3744 break;
3745 }
3746 case ISD::UMUL_LOHI:
3747 case ISD::SMUL_LOHI: {
3748 SDValue LHS = Node->getOperand(0);
3749 SDValue RHS = Node->getOperand(1);
3750 MVT VT = LHS.getSimpleValueType();
3751 unsigned MULHOpcode =
3752 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3753
3754 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3755 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3756 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3757 break;
3758 }
3759
3761 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3762 assert(TLI.isTypeLegal(HalfType));
3763 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3764 HalfType, DAG,
3765 TargetLowering::MulExpansionKind::Always)) {
3766 for (unsigned i = 0; i < 2; ++i) {
3767 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3768 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3769 SDValue Shift = DAG.getConstant(
3770 HalfType.getScalarSizeInBits(), dl,
3771 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3772 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3773 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3774 }
3775 break;
3776 }
3777 break;
3778 }
3779 case ISD::MUL: {
3780 EVT VT = Node->getValueType(0);
3781 SDVTList VTs = DAG.getVTList(VT, VT);
3782 // See if multiply or divide can be lowered using two-result operations.
3783 // We just need the low half of the multiply; try both the signed
3784 // and unsigned forms. If the target supports both SMUL_LOHI and
3785 // UMUL_LOHI, form a preference by checking which forms of plain
3786 // MULH it supports.
3787 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3788 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3789 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3790 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3791 unsigned OpToUse = 0;
3792 if (HasSMUL_LOHI && !HasMULHS) {
3793 OpToUse = ISD::SMUL_LOHI;
3794 } else if (HasUMUL_LOHI && !HasMULHU) {
3795 OpToUse = ISD::UMUL_LOHI;
3796 } else if (HasSMUL_LOHI) {
3797 OpToUse = ISD::SMUL_LOHI;
3798 } else if (HasUMUL_LOHI) {
3799 OpToUse = ISD::UMUL_LOHI;
3800 }
3801 if (OpToUse) {
3802 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3803 Node->getOperand(1)));
3804 break;
3805 }
3806
3807 SDValue Lo, Hi;
3808 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3809 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3810 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3811 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3812 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3813 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3814 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3815 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3816 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3817 SDValue Shift =
3818 DAG.getConstant(HalfType.getSizeInBits(), dl,
3819 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3820 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3821 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3822 }
3823 break;
3824 }
3825 case ISD::FSHL:
3826 case ISD::FSHR:
3827 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3828 Results.push_back(Expanded);
3829 break;
3830 case ISD::ROTL:
3831 case ISD::ROTR:
3832 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3833 Results.push_back(Expanded);
3834 break;
3835 case ISD::SADDSAT:
3836 case ISD::UADDSAT:
3837 case ISD::SSUBSAT:
3838 case ISD::USUBSAT:
3839 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3840 break;
3841 case ISD::SSHLSAT:
3842 case ISD::USHLSAT:
3843 Results.push_back(TLI.expandShlSat(Node, DAG));
3844 break;
3845 case ISD::SMULFIX:
3846 case ISD::SMULFIXSAT:
3847 case ISD::UMULFIX:
3848 case ISD::UMULFIXSAT:
3849 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3850 break;
3851 case ISD::SDIVFIX:
3852 case ISD::SDIVFIXSAT:
3853 case ISD::UDIVFIX:
3854 case ISD::UDIVFIXSAT:
3855 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3856 Node->getOperand(0),
3857 Node->getOperand(1),
3858 Node->getConstantOperandVal(2),
3859 DAG)) {
3860 Results.push_back(V);
3861 break;
3862 }
3863 // FIXME: We might want to retry here with a wider type if we fail, if that
3864 // type is legal.
3865 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3866 // <= 128 (which is the case for all of the default Embedded-C types),
3867 // we will only get here with types and scales that we could always expand
3868 // if we were allowed to generate libcalls to division functions of illegal
3869 // type. But we cannot do that.
3870 llvm_unreachable("Cannot expand DIVFIX!");
3871 case ISD::UADDO_CARRY:
3872 case ISD::USUBO_CARRY: {
3873 SDValue LHS = Node->getOperand(0);
3874 SDValue RHS = Node->getOperand(1);
3875 SDValue Carry = Node->getOperand(2);
3876
3877 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3878
3879 // Initial add of the 2 operands.
3880 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3881 EVT VT = LHS.getValueType();
3882 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3883
3884 // Initial check for overflow.
3885 EVT CarryType = Node->getValueType(1);
3886 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3888 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3889
3890 // Add of the sum and the carry.
3891 SDValue One = DAG.getConstant(1, dl, VT);
3892 SDValue CarryExt =
3893 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3894 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3895
3896 // Second check for overflow. If we are adding, we can only overflow if the
3897 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3898 // If we are subtracting, we can only overflow if the initial sum is 0 and
3899 // the carry is set, resulting in a new sum of all 1s.
3900 SDValue Zero = DAG.getConstant(0, dl, VT);
3901 SDValue Overflow2 =
3902 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3903 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3904 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3905 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3906
3907 SDValue ResultCarry =
3908 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3909
3910 Results.push_back(Sum2);
3911 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3912 break;
3913 }
3914 case ISD::SADDO:
3915 case ISD::SSUBO: {
3916 SDValue Result, Overflow;
3917 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3918 Results.push_back(Result);
3919 Results.push_back(Overflow);
3920 break;
3921 }
3922 case ISD::UADDO:
3923 case ISD::USUBO: {
3924 SDValue Result, Overflow;
3925 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3926 Results.push_back(Result);
3927 Results.push_back(Overflow);
3928 break;
3929 }
3930 case ISD::UMULO:
3931 case ISD::SMULO: {
3932 SDValue Result, Overflow;
3933 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3934 Results.push_back(Result);
3935 Results.push_back(Overflow);
3936 }
3937 break;
3938 }
3939 case ISD::BUILD_PAIR: {
3940 EVT PairTy = Node->getValueType(0);
3941 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3942 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3943 Tmp2 = DAG.getNode(
3944 ISD::SHL, dl, PairTy, Tmp2,
3945 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3946 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3947 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3948 break;
3949 }
3950 case ISD::SELECT:
3951 Tmp1 = Node->getOperand(0);
3952 Tmp2 = Node->getOperand(1);
3953 Tmp3 = Node->getOperand(2);
3954 if (Tmp1.getOpcode() == ISD::SETCC) {
3955 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3956 Tmp2, Tmp3,
3957 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3958 } else {
3959 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3960 DAG.getConstant(0, dl, Tmp1.getValueType()),
3961 Tmp2, Tmp3, ISD::SETNE);
3962 }
3963 Tmp1->setFlags(Node->getFlags());
3964 Results.push_back(Tmp1);
3965 break;
3966 case ISD::BR_JT: {
3967 SDValue Chain = Node->getOperand(0);
3968 SDValue Table = Node->getOperand(1);
3969 SDValue Index = Node->getOperand(2);
3970 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3971
3972 const DataLayout &TD = DAG.getDataLayout();
3973 EVT PTy = TLI.getPointerTy(TD);
3974
3975 unsigned EntrySize =
3976 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3977
3978 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3979 // This transformation needs to be done here since otherwise the MIPS
3980 // backend will end up emitting a three instruction multiply sequence
3981 // instead of a single shift and MSP430 will call a runtime function.
3982 if (llvm::isPowerOf2_32(EntrySize))
3983 Index = DAG.getNode(
3984 ISD::SHL, dl, Index.getValueType(), Index,
3985 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3986 else
3987 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3988 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3989 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3990 Index, Table);
3991
3992 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3993 SDValue LD = DAG.getExtLoad(
3994 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3995 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3996 Addr = LD;
3997 if (TLI.isJumpTableRelative()) {
3998 // For PIC, the sequence is:
3999 // BRIND(load(Jumptable + index) + RelocBase)
4000 // RelocBase can be JumpTable, GOT or some sort of global base.
4001 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
4002 TLI.getPICJumpTableRelocBase(Table, DAG));
4003 }
4004
4005 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
4006 Results.push_back(Tmp1);
4007 break;
4008 }
4009 case ISD::BRCOND:
4010 // Expand brcond's setcc into its constituent parts and create a BR_CC
4011 // Node.
4012 Tmp1 = Node->getOperand(0);
4013 Tmp2 = Node->getOperand(1);
4014 if (Tmp2.getOpcode() == ISD::SETCC &&
4015 TLI.isOperationLegalOrCustom(ISD::BR_CC,
4016 Tmp2.getOperand(0).getValueType())) {
4017 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
4018 Tmp2.getOperand(0), Tmp2.getOperand(1),
4019 Node->getOperand(2));
4020 } else {
4021 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4022 if (Tmp2.isUndef() ||
4023 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
4024 Tmp3 = Tmp2;
4025 else
4026 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
4027 DAG.getConstant(1, dl, Tmp2.getValueType()));
4028 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4029 DAG.getCondCode(ISD::SETNE), Tmp3,
4030 DAG.getConstant(0, dl, Tmp3.getValueType()),
4031 Node->getOperand(2));
4032 }
4033 Results.push_back(Tmp1);
4034 break;
4035 case ISD::SETCC:
4036 case ISD::VP_SETCC:
4037 case ISD::STRICT_FSETCC:
4038 case ISD::STRICT_FSETCCS: {
4039 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4040 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4041 Node->getOpcode() == ISD::STRICT_FSETCCS;
4042 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4043 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4044 unsigned Offset = IsStrict ? 1 : 0;
4045 Tmp1 = Node->getOperand(0 + Offset);
4046 Tmp2 = Node->getOperand(1 + Offset);
4047 Tmp3 = Node->getOperand(2 + Offset);
4048 SDValue Mask, EVL;
4049 if (IsVP) {
4050 Mask = Node->getOperand(3 + Offset);
4051 EVL = Node->getOperand(4 + Offset);
4052 }
4053 bool Legalized = TLI.LegalizeSetCCCondCode(
4054 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4055 Chain, IsSignaling);
4056
4057 if (Legalized) {
4058 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4059 // condition code, create a new SETCC node.
4060 if (Tmp3.getNode()) {
4061 if (IsStrict) {
4062 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4063 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4064 Chain = Tmp1.getValue(1);
4065 } else if (IsVP) {
4066 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4067 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4068 } else {
4069 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4070 Tmp2, Tmp3, Node->getFlags());
4071 }
4072 }
4073
4074 // If we expanded the SETCC by inverting the condition code, then wrap
4075 // the existing SETCC in a NOT to restore the intended condition.
4076 if (NeedInvert) {
4077 if (!IsVP)
4078 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4079 else
4080 Tmp1 =
4081 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4082 }
4083
4084 Results.push_back(Tmp1);
4085 if (IsStrict)
4086 Results.push_back(Chain);
4087
4088 break;
4089 }
4090
4091 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4092 // understand if this code is useful for strict nodes.
4093 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4094
4095 // Otherwise, SETCC for the given comparison type must be completely
4096 // illegal; expand it into a SELECT_CC.
4097 // FIXME: This drops the mask/evl for VP_SETCC.
4098 EVT VT = Node->getValueType(0);
4099 EVT Tmp1VT = Tmp1.getValueType();
4100 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4101 DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4102 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4103 Tmp1->setFlags(Node->getFlags());
4104 Results.push_back(Tmp1);
4105 break;
4106 }
4107 case ISD::SELECT_CC: {
4108 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4109 Tmp1 = Node->getOperand(0); // LHS
4110 Tmp2 = Node->getOperand(1); // RHS
4111 Tmp3 = Node->getOperand(2); // True
4112 Tmp4 = Node->getOperand(3); // False
4113 EVT VT = Node->getValueType(0);
4114 SDValue Chain;
4115 SDValue CC = Node->getOperand(4);
4116 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4117
4118 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4119 // If the condition code is legal, then we need to expand this
4120 // node using SETCC and SELECT.
4121 EVT CmpVT = Tmp1.getValueType();
4122 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4123 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4124 "expanded.");
4125 EVT CCVT = getSetCCResultType(CmpVT);
4126 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4127 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4128 break;
4129 }
4130
4131 // SELECT_CC is legal, so the condition code must not be.
4132 bool Legalized = false;
4133 // Try to legalize by inverting the condition. This is for targets that
4134 // might support an ordered version of a condition, but not the unordered
4135 // version (or vice versa).
4136 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4137 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4138 // Use the new condition code and swap true and false
4139 Legalized = true;
4140 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4141 Tmp1->setFlags(Node->getFlags());
4142 } else {
4143 // If The inverse is not legal, then try to swap the arguments using
4144 // the inverse condition code.
4146 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4147 // The swapped inverse condition is legal, so swap true and false,
4148 // lhs and rhs.
4149 Legalized = true;
4150 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4151 Tmp1->setFlags(Node->getFlags());
4152 }
4153 }
4154
4155 if (!Legalized) {
4156 Legalized = TLI.LegalizeSetCCCondCode(
4157 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4158 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4159
4160 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4161
4162 // If we expanded the SETCC by inverting the condition code, then swap
4163 // the True/False operands to match.
4164 if (NeedInvert)
4165 std::swap(Tmp3, Tmp4);
4166
4167 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4168 // condition code, create a new SELECT_CC node.
4169 if (CC.getNode()) {
4170 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4171 Tmp1, Tmp2, Tmp3, Tmp4, CC);
4172 } else {
4173 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4174 CC = DAG.getCondCode(ISD::SETNE);
4175 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4176 Tmp2, Tmp3, Tmp4, CC);
4177 }
4178 Tmp1->setFlags(Node->getFlags());
4179 }
4180 Results.push_back(Tmp1);
4181 break;
4182 }
4183 case ISD::BR_CC: {
4184 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4185 SDValue Chain;
4186 Tmp1 = Node->getOperand(0); // Chain
4187 Tmp2 = Node->getOperand(2); // LHS
4188 Tmp3 = Node->getOperand(3); // RHS
4189 Tmp4 = Node->getOperand(1); // CC
4190
4191 bool Legalized = TLI.LegalizeSetCCCondCode(
4192 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4193 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4194 (void)Legalized;
4195 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4196
4197 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4198 // node.
4199 if (Tmp4.getNode()) {
4200 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4201
4202 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4203 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4204 } else {
4205 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4206 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4207 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4208 Tmp2, Tmp3, Node->getOperand(4));
4209 }
4210 Results.push_back(Tmp1);
4211 break;
4212 }
4213 case ISD::BUILD_VECTOR:
4214 Results.push_back(ExpandBUILD_VECTOR(Node));
4215 break;
4216 case ISD::SPLAT_VECTOR:
4217 Results.push_back(ExpandSPLAT_VECTOR(Node));
4218 break;
4219 case ISD::SRA:
4220 case ISD::SRL:
4221 case ISD::SHL: {
4222 // Scalarize vector SRA/SRL/SHL.
4223 EVT VT = Node->getValueType(0);
4224 assert(VT.isVector() && "Unable to legalize non-vector shift");
4225 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4226 unsigned NumElem = VT.getVectorNumElements();
4227
4229 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4230 SDValue Ex =
4232 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4233 SDValue Sh =
4235 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4236 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4237 VT.getScalarType(), Ex, Sh));
4238 }
4239
4240 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4241 Results.push_back(Result);
4242 break;
4243 }
4246 case ISD::VECREDUCE_ADD:
4247 case ISD::VECREDUCE_MUL:
4248 case ISD::VECREDUCE_AND:
4249 case ISD::VECREDUCE_OR:
4250 case ISD::VECREDUCE_XOR:
4259 Results.push_back(TLI.expandVecReduce(Node, DAG));
4260 break;
4262 case ISD::GlobalAddress:
4265 case ISD::ConstantPool:
4266 case ISD::JumpTable:
4270 // FIXME: Custom lowering for these operations shouldn't return null!
4271 // Return true so that we don't call ConvertNodeToLibcall which also won't
4272 // do anything.
4273 return true;
4274 }
4275
4276 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4277 // FIXME: We were asked to expand a strict floating-point operation,
4278 // but there is currently no expansion implemented that would preserve
4279 // the "strict" properties. For now, we just fall back to the non-strict
4280 // version if that is legal on the target. The actual mutation of the
4281 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4282 switch (Node->getOpcode()) {
4283 default:
4284 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4285 Node->getValueType(0))
4286 == TargetLowering::Legal)
4287 return true;
4288 break;
4289 case ISD::STRICT_FSUB: {
4290 if (TLI.getStrictFPOperationAction(
4291 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4292 return true;
4293 if (TLI.getStrictFPOperationAction(
4294 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4295 break;
4296
4297 EVT VT = Node->getValueType(0);
4298 const SDNodeFlags Flags = Node->getFlags();
4299 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4300 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4301 {Node->getOperand(0), Node->getOperand(1), Neg},
4302 Flags);
4303
4304 Results.push_back(Fadd);
4305 Results.push_back(Fadd.getValue(1));
4306 break;
4307 }
4310 case ISD::STRICT_LRINT:
4311 case ISD::STRICT_LLRINT:
4312 case ISD::STRICT_LROUND:
4314 // These are registered by the operand type instead of the value
4315 // type. Reflect that here.
4316 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4317 Node->getOperand(1).getValueType())
4318 == TargetLowering::Legal)
4319 return true;
4320 break;
4321 }
4322 }
4323
4324 // Replace the original node with the legalized result.
4325 if (Results.empty()) {
4326 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4327 return false;
4328 }
4329
4330 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4331 ReplaceNode(Node, Results.data());
4332 return true;
4333}
4334
4335void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4336 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4338 SDLoc dl(Node);
4339 // FIXME: Check flags on the node to see if we can use a finite call.
4340 unsigned Opc = Node->getOpcode();
4341 switch (Opc) {
4342 case ISD::ATOMIC_FENCE: {
4343 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4344 // FIXME: handle "fence singlethread" more efficiently.
4346
4348 CLI.setDebugLoc(dl)
4349 .setChain(Node->getOperand(0))
4350 .setLibCallee(
4351 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4352 DAG.getExternalSymbol("__sync_synchronize",
4353 TLI.getPointerTy(DAG.getDataLayout())),
4354 std::move(Args));
4355
4356 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4357
4358 Results.push_back(CallResult.second);
4359 break;
4360 }
4361 // By default, atomic intrinsics are marked Legal and lowered. Targets
4362 // which don't support them directly, however, may want libcalls, in which
4363 // case they mark them Expand, and we get here.
4364 case ISD::ATOMIC_SWAP:
4376 case ISD::ATOMIC_CMP_SWAP: {
4377 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4378 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4379 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4380 EVT RetVT = Node->getValueType(0);
4383 if (TLI.getLibcallName(LC)) {
4384 // If outline atomic available, prepare its arguments and expand.
4385 Ops.append(Node->op_begin() + 2, Node->op_end());
4386 Ops.push_back(Node->getOperand(1));
4387
4388 } else {
4389 LC = RTLIB::getSYNC(Opc, VT);
4390 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4391 "Unexpected atomic op or value type!");
4392 // Arguments for expansion to sync libcall
4393 Ops.append(Node->op_begin() + 1, Node->op_end());
4394 }
4395 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4396 Ops, CallOptions,
4397 SDLoc(Node),
4398 Node->getOperand(0));
4399 Results.push_back(Tmp.first);
4400 Results.push_back(Tmp.second);
4401 break;
4402 }
4403 case ISD::TRAP: {
4404 // If this operation is not supported, lower it to 'abort()' call
4407 CLI.setDebugLoc(dl)
4408 .setChain(Node->getOperand(0))
4409 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4410 DAG.getExternalSymbol(
4411 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4412 std::move(Args));
4413 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4414
4415 Results.push_back(CallResult.second);
4416 break;
4417 }
4418 case ISD::FMINNUM:
4420 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4421 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4422 RTLIB::FMIN_PPCF128, Results);
4423 break;
4424 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4425 // libcall legalization for these nodes, but there is no default expasion for
4426 // these nodes either (see PR63267 for example).
4427 case ISD::FMAXNUM:
4429 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4430 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4431 RTLIB::FMAX_PPCF128, Results);
4432 break;
4433 case ISD::FSQRT:
4434 case ISD::STRICT_FSQRT:
4435 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4436 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4437 RTLIB::SQRT_PPCF128, Results);
4438 break;
4439 case ISD::FCBRT:
4440 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4441 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4442 RTLIB::CBRT_PPCF128, Results);
4443 break;
4444 case ISD::FSIN:
4445 case ISD::STRICT_FSIN:
4446 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4447 RTLIB::SIN_F80, RTLIB::SIN_F128,
4448 RTLIB::SIN_PPCF128, Results);
4449 break;
4450 case ISD::FCOS:
4451 case ISD::STRICT_FCOS:
4452 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4453 RTLIB::COS_F80, RTLIB::COS_F128,
4454 RTLIB::COS_PPCF128, Results);
4455 break;
4456 case ISD::FSINCOS:
4457 // Expand into sincos libcall.
4458 ExpandSinCosLibCall(Node, Results);
4459 break;
4460 case ISD::FLOG:
4461 case ISD::STRICT_FLOG:
4462 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4463 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4464 break;
4465 case ISD::FLOG2:
4466 case ISD::STRICT_FLOG2:
4467 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4468 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4469 break;
4470 case ISD::FLOG10:
4471 case ISD::STRICT_FLOG10:
4472 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4473 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4474 break;
4475 case ISD::FEXP:
4476 case ISD::STRICT_FEXP:
4477 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4478 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4479 break;
4480 case ISD::FEXP2:
4481 case ISD::STRICT_FEXP2:
4482 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4483 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4484 break;
4485 case ISD::FEXP10:
4486 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4487 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4488 break;
4489 case ISD::FTRUNC:
4490 case ISD::STRICT_FTRUNC:
4491 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4492 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4493 RTLIB::TRUNC_PPCF128, Results);
4494 break;
4495 case ISD::FFLOOR:
4496 case ISD::STRICT_FFLOOR:
4497 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4498 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4499 RTLIB::FLOOR_PPCF128, Results);
4500 break;
4501 case ISD::FCEIL:
4502 case ISD::STRICT_FCEIL:
4503 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4504 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4505 RTLIB::CEIL_PPCF128, Results);
4506 break;
4507 case ISD::FRINT:
4508 case ISD::STRICT_FRINT:
4509 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4510 RTLIB::RINT_F80, RTLIB::RINT_F128,
4511 RTLIB::RINT_PPCF128, Results);
4512 break;
4513 case ISD::FNEARBYINT:
4515 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4516 RTLIB::NEARBYINT_F64,
4517 RTLIB::NEARBYINT_F80,
4518 RTLIB::NEARBYINT_F128,
4519 RTLIB::NEARBYINT_PPCF128, Results);
4520 break;
4521 case ISD::FROUND:
4522 case ISD::STRICT_FROUND:
4523 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4524 RTLIB::ROUND_F64,
4525 RTLIB::ROUND_F80,
4526 RTLIB::ROUND_F128,
4527 RTLIB::ROUND_PPCF128, Results);
4528 break;
4529 case ISD::FROUNDEVEN:
4531 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4532 RTLIB::ROUNDEVEN_F64,
4533 RTLIB::ROUNDEVEN_F80,
4534 RTLIB::ROUNDEVEN_F128,
4535 RTLIB::ROUNDEVEN_PPCF128, Results);
4536 break;
4537 case ISD::FLDEXP:
4538 case ISD::STRICT_FLDEXP:
4539 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4540 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4541 break;
4542 case ISD::FFREXP: {
4543 ExpandFrexpLibCall(Node, Results);
4544 break;
4545 }
4546 case ISD::FPOWI:
4547 case ISD::STRICT_FPOWI: {
4548 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4549 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4550 if (!TLI.getLibcallName(LC)) {
4551 // Some targets don't have a powi libcall; use pow instead.
4552 if (Node->isStrictFPOpcode()) {
4554 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4555 {Node->getValueType(0), Node->getValueType(1)},
4556 {Node->getOperand(0), Node->getOperand(2)});
4557 SDValue FPOW =
4558 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4559 {Node->getValueType(0), Node->getValueType(1)},
4560 {Exponent.getValue(1), Node->getOperand(1), Exponent});
4561 Results.push_back(FPOW);
4562 Results.push_back(FPOW.getValue(1));
4563 } else {
4565 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4566 Node->getOperand(1));
4567 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4568 Node->getValueType(0),
4569 Node->getOperand(0), Exponent));
4570 }
4571 break;
4572 }
4573 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4574 bool ExponentHasSizeOfInt =
4575 DAG.getLibInfo().getIntSize() ==
4576 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4577 if (!ExponentHasSizeOfInt) {
4578 // If the exponent does not match with sizeof(int) a libcall to
4579 // RTLIB::POWI would use the wrong type for the argument.
4580 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4581 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4582 break;
4583 }
4584 ExpandFPLibCall(Node, LC, Results);
4585 break;
4586 }
4587 case ISD::FPOW:
4588 case ISD::STRICT_FPOW:
4589 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4590 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4591 break;
4592 case ISD::LROUND:
4593 case ISD::STRICT_LROUND:
4594 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4595 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4596 RTLIB::LROUND_F128,
4597 RTLIB::LROUND_PPCF128, Results);
4598 break;
4599 case ISD::LLROUND:
4601 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4602 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4603 RTLIB::LLROUND_F128,
4604 RTLIB::LLROUND_PPCF128, Results);
4605 break;
4606 case ISD::LRINT:
4607 case ISD::STRICT_LRINT:
4608 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4609 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4610 RTLIB::LRINT_F128,
4611 RTLIB::LRINT_PPCF128, Results);
4612 break;
4613 case ISD::LLRINT:
4614 case ISD::STRICT_LLRINT:
4615 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4616 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4617 RTLIB::LLRINT_F128,
4618 RTLIB::LLRINT_PPCF128, Results);
4619 break;
4620 case ISD::FDIV:
4621 case ISD::STRICT_FDIV:
4622 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4623 RTLIB::DIV_F80, RTLIB::DIV_F128,
4624 RTLIB::DIV_PPCF128, Results);
4625 break;
4626 case ISD::FREM:
4627 case ISD::STRICT_FREM:
4628 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4629 RTLIB::REM_F80, RTLIB::REM_F128,
4630 RTLIB::REM_PPCF128, Results);
4631 break;
4632 case ISD::FMA:
4633 case ISD::STRICT_FMA:
4634 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4635 RTLIB::FMA_F80, RTLIB::FMA_F128,
4636 RTLIB::FMA_PPCF128, Results);
4637 break;
4638 case ISD::FADD:
4639 case ISD::STRICT_FADD:
4640 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4641 RTLIB::ADD_F80, RTLIB::ADD_F128,
4642 RTLIB::ADD_PPCF128, Results);
4643 break;
4644 case ISD::FMUL:
4645 case ISD::STRICT_FMUL:
4646 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4647 RTLIB::MUL_F80, RTLIB::MUL_F128,
4648 RTLIB::MUL_PPCF128, Results);
4649 break;
4650 case ISD::FP16_TO_FP:
4651 if (Node->getValueType(0) == MVT::f32) {
4652 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
4653 }
4654 break;
4656 if (Node->getValueType(0) == MVT::f32) {
4658 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4659 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1),
4660 CallOptions, SDLoc(Node), Node->getOperand(0));
4661 Results.push_back(Tmp.first);
4662 Results.push_back(Tmp.second);
4663 }
4664 break;
4666 if (Node->getValueType(0) == MVT::f32) {
4668 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4669 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4670 SDLoc(Node), Node->getOperand(0));
4671 Results.push_back(Tmp.first);
4672 Results.push_back(Tmp.second);
4673 }
4674 break;
4675 }
4676 case ISD::FP_TO_FP16: {
4677 RTLIB::Libcall LC =
4678 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4679 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4680 Results.push_back(ExpandLibCall(LC, Node, false).first);
4681 break;
4682 }
4683 case ISD::FP_TO_BF16: {
4684 RTLIB::Libcall LC =
4685 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4686 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4687 Results.push_back(ExpandLibCall(LC, Node, false).first);
4688 break;
4689 }
4692 case ISD::SINT_TO_FP:
4693 case ISD::UINT_TO_FP: {
4694 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4695 bool IsStrict = Node->isStrictFPOpcode();
4696 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4697 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4698 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4699 EVT RVT = Node->getValueType(0);
4700 EVT NVT = EVT();
4701 SDLoc dl(Node);
4702
4703 // Even if the input is legal, no libcall may exactly match, eg. we don't
4704 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4705 // eg: i13 -> fp. Then, look for an appropriate libcall.
4706 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4707 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4708 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4709 ++t) {
4710 NVT = (MVT::SimpleValueType)t;
4711 // The source needs to big enough to hold the operand.
4712 if (NVT.bitsGE(SVT))
4713 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4714 : RTLIB::getUINTTOFP(NVT, RVT);
4715 }
4716 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4717
4718 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4719 // Sign/zero extend the argument if the libcall takes a larger type.
4720 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4721 NVT, Node->getOperand(IsStrict ? 1 : 0));
4723 CallOptions.setSExt(Signed);
4724 std::pair<SDValue, SDValue> Tmp =
4725 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4726 Results.push_back(Tmp.first);
4727 if (IsStrict)
4728 Results.push_back(Tmp.second);
4729 break;
4730 }
4731 case ISD::FP_TO_SINT:
4732 case ISD::FP_TO_UINT:
4735 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4736 bool IsStrict = Node->isStrictFPOpcode();
4737 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4738 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4739
4740 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4741 EVT SVT = Op.getValueType();
4742 EVT RVT = Node->getValueType(0);
4743 EVT NVT = EVT();
4744 SDLoc dl(Node);
4745
4746 // Even if the result is legal, no libcall may exactly match, eg. we don't
4747 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4748 // eg: fp -> i32. Then, look for an appropriate libcall.
4749 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4750 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4751 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4752 ++IntVT) {
4753 NVT = (MVT::SimpleValueType)IntVT;
4754 // The type needs to big enough to hold the result.
4755 if (NVT.bitsGE(RVT))
4756 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4757 : RTLIB::getFPTOUINT(SVT, NVT);
4758 }
4759 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4760
4761 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4763 std::pair<SDValue, SDValue> Tmp =
4764 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4765
4766 // Truncate the result if the libcall returns a larger type.
4767 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4768 if (IsStrict)
4769 Results.push_back(Tmp.second);
4770 break;
4771 }
4772
4773 case ISD::FP_ROUND:
4774 case ISD::STRICT_FP_ROUND: {
4775 // X = FP_ROUND(Y, TRUNC)
4776 // TRUNC is a flag, which is always an integer that is zero or one.
4777 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4778 // is known to not change the value of Y.
4779 // We can only expand it into libcall if the TRUNC is 0.
4780 bool IsStrict = Node->isStrictFPOpcode();
4781 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4782 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4783 EVT VT = Node->getValueType(0);
4784 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4785 "Unable to expand as libcall if it is not normal rounding");
4786
4787 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4788 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4789
4791 std::pair<SDValue, SDValue> Tmp =
4792 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4793 Results.push_back(Tmp.first);
4794 if (IsStrict)
4795 Results.push_back(Tmp.second);
4796 break;
4797 }
4798 case ISD::FP_EXTEND: {
4799 Results.push_back(
4800 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4801 Node->getValueType(0)),
4802 Node, false).first);
4803 break;
4804 }
4808 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4809 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
4810 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16);
4811 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
4812 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::bf16);
4813 else
4814 LC = RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4815 Node->getValueType(0));
4816
4817 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4818
4820 std::pair<SDValue, SDValue> Tmp =
4821 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4822 CallOptions, SDLoc(Node), Node->getOperand(0));
4823 Results.push_back(Tmp.first);
4824 Results.push_back(Tmp.second);
4825 break;
4826 }
4827 case ISD::FSUB:
4828 case ISD::STRICT_FSUB:
4829 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4830 RTLIB::SUB_F80, RTLIB::SUB_F128,
4831 RTLIB::SUB_PPCF128, Results);
4832 break;
4833 case ISD::SREM:
4834 Results.push_back(ExpandIntLibCall(Node, true,
4835 RTLIB::SREM_I8,
4836 RTLIB::SREM_I16, RTLIB::SREM_I32,
4837 RTLIB::SREM_I64, RTLIB::SREM_I128));
4838 break;
4839 case ISD::UREM:
4840 Results.push_back(ExpandIntLibCall(Node, false,
4841 RTLIB::UREM_I8,
4842 RTLIB::UREM_I16, RTLIB::UREM_I32,
4843 RTLIB::UREM_I64, RTLIB::UREM_I128));
4844 break;
4845 case ISD::SDIV:
4846 Results.push_back(ExpandIntLibCall(Node, true,
4847 RTLIB::SDIV_I8,
4848 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4849 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4850 break;
4851 case ISD::UDIV:
4852 Results.push_back(ExpandIntLibCall(Node, false,
4853 RTLIB::UDIV_I8,
4854 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4855 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4856 break;
4857 case ISD::SDIVREM:
4858 case ISD::UDIVREM:
4859 // Expand into divrem libcall
4860 ExpandDivRemLibCall(Node, Results);
4861 break;
4862 case ISD::MUL:
4863 Results.push_back(ExpandIntLibCall(Node, false,
4864 RTLIB::MUL_I8,
4865 RTLIB::MUL_I16, RTLIB::MUL_I32,
4866 RTLIB::MUL_I64, RTLIB::MUL_I128));
4867 break;
4869 switch (Node->getSimpleValueType(0).SimpleTy) {
4870 default:
4871 llvm_unreachable("LibCall explicitly requested, but not available");
4872 case MVT::i32:
4873 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first);
4874 break;
4875 case MVT::i64:
4876 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first);
4877 break;
4878 case MVT::i128:
4879 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first);
4880 break;
4881 }
4882 break;
4883 case ISD::RESET_FPENV: {
4884 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
4885 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
4886 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4887 SDValue Chain = Node->getOperand(0);
4888 Results.push_back(
4889 DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));
4890 break;
4891 }
4892 case ISD::GET_FPENV_MEM: {
4893 SDValue Chain = Node->getOperand(0);
4894 SDValue EnvPtr = Node->getOperand(1);
4895 Results.push_back(
4896 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4897 break;
4898 }
4899 case ISD::SET_FPENV_MEM: {
4900 SDValue Chain = Node->getOperand(0);
4901 SDValue EnvPtr = Node->getOperand(1);
4902 Results.push_back(
4903 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4904 break;
4905 }
4906 case ISD::GET_FPMODE: {
4907 // Call fegetmode, which saves control modes into a stack slot. Then load
4908 // the value to return from the stack.
4909 EVT ModeVT = Node->getValueType(0);
4910 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4911 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4912 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4913 Node->getOperand(0), dl);
4914 SDValue LdInst = DAG.getLoad(
4915 ModeVT, dl, Chain, StackPtr,
4916 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4917 Results.push_back(LdInst);
4918 Results.push_back(LdInst.getValue(1));
4919 break;
4920 }
4921 case ISD::SET_FPMODE: {
4922 // Move control modes to stack slot and then call fesetmode with the pointer
4923 // to the slot as argument.
4924 SDValue Mode = Node->getOperand(1);
4925 EVT ModeVT = Mode.getValueType();
4926 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4927 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4928 SDValue StInst = DAG.getStore(
4929 Node->getOperand(0), dl, Mode, StackPtr,
4930 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4931 Results.push_back(
4932 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4933 break;
4934 }
4935 case ISD::RESET_FPMODE: {
4936 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
4937 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
4938 // target must provide custom lowering.
4939 const DataLayout &DL = DAG.getDataLayout();
4940 EVT PtrTy = TLI.getPointerTy(DL);
4941 SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy);
4942 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4943 Node->getOperand(0), dl));
4944 break;
4945 }
4946 }
4947
4948 // Replace the original node with the legalized result.
4949 if (!Results.empty()) {
4950 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4951 ReplaceNode(Node, Results.data());
4952 } else
4953 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4954}
4955
4956// Determine the vector type to use in place of an original scalar element when
4957// promoting equally sized vectors.
4959 MVT EltVT, MVT NewEltVT) {
4960 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4961 MVT MidVT = OldEltsPerNewElt == 1
4962 ? NewEltVT
4963 : MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4964 assert(TLI.isTypeLegal(MidVT) && "unexpected");
4965 return MidVT;
4966}
4967
4968void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4969 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4971 MVT OVT = Node->getSimpleValueType(0);
4972 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4973 Node->getOpcode() == ISD::SINT_TO_FP ||
4974 Node->getOpcode() == ISD::SETCC ||
4975 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4976 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4977 OVT = Node->getOperand(0).getSimpleValueType();
4978 }
4979 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4980 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4981 Node->getOpcode() == ISD::STRICT_FSETCC ||
4982 Node->getOpcode() == ISD::STRICT_FSETCCS)
4983 OVT = Node->getOperand(1).getSimpleValueType();
4984 if (Node->getOpcode() == ISD::BR_CC ||
4985 Node->getOpcode() == ISD::SELECT_CC)
4986 OVT = Node->getOperand(2).getSimpleValueType();
4987 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4988 SDLoc dl(Node);
4989 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4990 switch (Node->getOpcode()) {
4991 case ISD::CTTZ:
4993 case ISD::CTLZ:
4995 case ISD::CTPOP:
4996 // Zero extend the argument unless its cttz, then use any_extend.
4997 if (Node->getOpcode() == ISD::CTTZ ||
4998 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4999 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
5000 else
5001 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
5002
5003 if (Node->getOpcode() == ISD::CTTZ) {
5004 // The count is the same in the promoted type except if the original
5005 // value was zero. This can be handled by setting the bit just off
5006 // the top of the original type.
5007 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
5008 OVT.getSizeInBits());
5009 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
5010 DAG.getConstant(TopBit, dl, NVT));
5011 }
5012 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5013 // already the correct result.
5014 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5015 if (Node->getOpcode() == ISD::CTLZ ||
5016 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
5017 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5018 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
5019 DAG.getConstant(NVT.getSizeInBits() -
5020 OVT.getSizeInBits(), dl, NVT));
5021 }
5022 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5023 break;
5024 case ISD::BITREVERSE:
5025 case ISD::BSWAP: {
5026 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5027 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
5028 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5029 Tmp1 = DAG.getNode(
5030 ISD::SRL, dl, NVT, Tmp1,
5031 DAG.getConstant(DiffBits, dl,
5032 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5033
5034 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5035 break;
5036 }
5037 case ISD::FP_TO_UINT:
5039 case ISD::FP_TO_SINT:
5041 PromoteLegalFP_TO_INT(Node, dl, Results);
5042 break;
5045 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
5046 break;
5047 case ISD::UINT_TO_FP:
5049 case ISD::SINT_TO_FP:
5051 PromoteLegalINT_TO_FP(Node, dl, Results);
5052 break;
5053 case ISD::VAARG: {
5054 SDValue Chain = Node->getOperand(0); // Get the chain.
5055 SDValue Ptr = Node->getOperand(1); // Get the pointer.
5056
5057 unsigned TruncOp;
5058 if (OVT.isVector()) {
5059 TruncOp = ISD::BITCAST;
5060 } else {
5061 assert(OVT.isInteger()
5062 && "VAARG promotion is supported only for vectors or integer types");
5063 TruncOp = ISD::TRUNCATE;
5064 }
5065
5066 // Perform the larger operation, then convert back
5067 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
5068 Node->getConstantOperandVal(3));
5069 Chain = Tmp1.getValue(1);
5070
5071 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
5072
5073 // Modified the chain result - switch anything that used the old chain to
5074 // use the new one.
5075 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
5076 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
5077 if (UpdatedNodes) {
5078 UpdatedNodes->insert(Tmp2.getNode());
5079 UpdatedNodes->insert(Chain.getNode());
5080 }
5081 ReplacedNode(Node);
5082 break;
5083 }
5084 case ISD::MUL:
5085 case ISD::SDIV:
5086 case ISD::SREM:
5087 case ISD::UDIV:
5088 case ISD::UREM:
5089 case ISD::SMIN:
5090 case ISD::SMAX:
5091 case ISD::UMIN:
5092 case ISD::UMAX:
5093 case ISD::AND:
5094 case ISD::OR:
5095 case ISD::XOR: {
5096 unsigned ExtOp, TruncOp;
5097 if (OVT.isVector()) {
5098 ExtOp = ISD::BITCAST;
5099 TruncOp = ISD::BITCAST;
5100 } else {
5101 assert(OVT.isInteger() && "Cannot promote logic operation");
5102
5103 switch (Node->getOpcode()) {
5104 default:
5105 ExtOp = ISD::ANY_EXTEND;
5106 break;
5107 case ISD::SDIV:
5108 case ISD::SREM:
5109 case ISD::SMIN:
5110 case ISD::SMAX:
5111 ExtOp = ISD::SIGN_EXTEND;
5112 break;
5113 case ISD::UDIV:
5114 case ISD::UREM:
5115 ExtOp = ISD::ZERO_EXTEND;
5116 break;
5117 case ISD::UMIN:
5118 case ISD::UMAX:
5119 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5120 ExtOp = ISD::SIGN_EXTEND;
5121 else
5122 ExtOp = ISD::ZERO_EXTEND;
5123 break;
5124 }
5125 TruncOp = ISD::TRUNCATE;
5126 }
5127 // Promote each of the values to the new type.
5128 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5129 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5130 // Perform the larger operation, then convert back
5131 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5132 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5133 break;
5134 }
5135 case ISD::UMUL_LOHI:
5136 case ISD::SMUL_LOHI: {
5137 // Promote to a multiply in a wider integer type.
5138 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5140 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5141 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5142 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
5143
5144 auto &DL = DAG.getDataLayout();
5145 unsigned OriginalSize = OVT.getScalarSizeInBits();
5146 Tmp2 = DAG.getNode(
5147 ISD::SRL, dl, NVT, Tmp1,
5148 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
5149 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5150 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5151 break;
5152 }
5153 case ISD::SELECT: {
5154 unsigned ExtOp, TruncOp;
5155 if (Node->getValueType(0).isVector() ||
5156 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
5157 ExtOp = ISD::BITCAST;
5158 TruncOp = ISD::BITCAST;
5159 } else if (Node->getValueType(0).isInteger()) {
5160 ExtOp = ISD::ANY_EXTEND;
5161 TruncOp = ISD::TRUNCATE;
5162 } else {
5163 ExtOp = ISD::FP_EXTEND;
5164 TruncOp = ISD::FP_ROUND;
5165 }
5166 Tmp1 = Node->getOperand(0);
5167 // Promote each of the values to the new type.
5168 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5169 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5170 // Perform the larger operation, then round down.
5171 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5172 Tmp1->setFlags(Node->getFlags());
5173 if (TruncOp != ISD::FP_ROUND)
5174 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
5175 else
5176 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
5177 DAG.getIntPtrConstant(0, dl));
5178 Results.push_back(Tmp1);
5179 break;
5180 }
5181 case ISD::VECTOR_SHUFFLE: {
5182 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
5183
5184 // Cast the two input vectors.
5185 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
5186 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
5187
5188 // Convert the shuffle mask to the right # elements.
5189 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5190 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
5191 Results.push_back(Tmp1);
5192 break;
5193 }
5194 case ISD::VECTOR_SPLICE: {
5195 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
5196 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
5197 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
5198 Node->getOperand(2));
5199 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
5200 break;
5201 }
5202 case ISD::SELECT_CC: {
5203 SDValue Cond = Node->getOperand(4);
5204 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
5205 // Type of the comparison operands.
5206 MVT CVT = Node->getSimpleValueType(0);
5207 assert(CVT == OVT && "not handled");
5208
5209 unsigned ExtOp = ISD::FP_EXTEND;
5210 if (NVT.isInteger()) {
5212 }
5213
5214 // Promote the comparison operands, if needed.
5215 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5216 Tmp1 = Node->getOperand(0);
5217 Tmp2 = Node->getOperand(1);
5218 } else {
5219 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5220 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5221 }
5222 // Cast the true/false operands.
5223 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5224 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5225
5226 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5227 Node->getFlags());
5228
5229 // Cast the result back to the original type.
5230 if (ExtOp != ISD::FP_EXTEND)
5231 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
5232 else
5233 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
5234 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
5235
5236 Results.push_back(Tmp1);
5237 break;
5238 }
5239 case ISD::SETCC:
5240 case ISD::STRICT_FSETCC:
5241 case ISD::STRICT_FSETCCS: {
5242 unsigned ExtOp = ISD::FP_EXTEND;
5243 if (NVT.isInteger()) {
5244 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
5245 if (isSignedIntSetCC(CCCode) ||
5246 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT))
5247 ExtOp = ISD::SIGN_EXTEND;
5248 else
5249 ExtOp = ISD::ZERO_EXTEND;
5250 }
5251 if (Node->isStrictFPOpcode()) {
5252 SDValue InChain = Node->getOperand(0);
5253 std::tie(Tmp1, std::ignore) =
5254 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
5255 std::tie(Tmp2, std::ignore) =
5256 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
5257 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
5258 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5259 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5260 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
5261 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5262 Node->getFlags()));
5263 Results.push_back(Results.back().getValue(1));
5264 break;
5265 }
5266 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5267 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5268 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
5269 Tmp2, Node->getOperand(2), Node->getFlags()));
5270 break;
5271 }
5272 case ISD::BR_CC: {
5273 unsigned ExtOp = ISD::FP_EXTEND;
5274 if (NVT.isInteger()) {
5275 ISD::CondCode CCCode =
5276 cast<CondCodeSDNode>(Node->getOperand(1))->get();
5278 }
5279 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5280 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5281 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
5282 Node->getOperand(0), Node->getOperand(1),
5283 Tmp1, Tmp2, Node->getOperand(4)));
5284 break;
5285 }
5286 case ISD::FADD:
5287 case ISD::FSUB:
5288 case ISD::FMUL:
5289 case ISD::FDIV:
5290 case ISD::FREM:
5291 case ISD::FMINNUM:
5292 case ISD::FMAXNUM:
5293 case ISD::FMINIMUM:
5294 case ISD::FMAXIMUM:
5295 case ISD::FPOW:
5296 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5297 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5298 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5299 Node->getFlags());
5300 Results.push_back(
5301 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5302 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5303 break;
5304 case ISD::STRICT_FADD:
5305 case ISD::STRICT_FSUB:
5306 case ISD::STRICT_FMUL:
5307 case ISD::STRICT_FDIV:
5310 case ISD::STRICT_FREM:
5311 case ISD::STRICT_FPOW:
5312 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5313 {Node->getOperand(0), Node->getOperand(1)});
5314 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5315 {Node->getOperand(0), Node->getOperand(2)});
5316 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5317 Tmp2.getValue(1));
5318 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5319 {Tmp3, Tmp1, Tmp2});
5320 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5321 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5322 Results.push_back(Tmp1);
5323 Results.push_back(Tmp1.getValue(1));
5324 break;
5325 case ISD::FMA:
5326 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5327 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5328 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
5329 Results.push_back(
5330 DAG.getNode(ISD::FP_ROUND, dl, OVT,
5331 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5332 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5333 break;
5334 case ISD::STRICT_FMA:
5335 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5336 {Node->getOperand(0), Node->getOperand(1)});
5337 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5338 {Node->getOperand(0), Node->getOperand(2)});
5339 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5340 {Node->getOperand(0), Node->getOperand(3)});
5341 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5342 Tmp2.getValue(1), Tmp3.getValue(1));
5343 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5344 {Tmp4, Tmp1, Tmp2, Tmp3});
5345 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5346 {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5347 Results.push_back(Tmp4);
5348 Results.push_back(Tmp4.getValue(1));
5349 break;
5350 case ISD::FCOPYSIGN:
5351 case ISD::FLDEXP:
5352 case ISD::FPOWI: {
5353 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5354 Tmp2 = Node->getOperand(1);
5355 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5356
5357 // fcopysign doesn't change anything but the sign bit, so
5358 // (fp_round (fcopysign (fpext a), b))
5359 // is as precise as
5360 // (fp_round (fpext a))
5361 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5362 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5363 Results.push_back(
5364 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5365 DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true)));
5366 break;
5367 }
5368 case ISD::STRICT_FPOWI:
5369 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5370 {Node->getOperand(0), Node->getOperand(1)});
5371 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5372 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5373 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5374 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5375 Results.push_back(Tmp3);
5376 Results.push_back(Tmp3.getValue(1));
5377 break;
5378 case ISD::FFREXP: {
5379 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5380 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1);
5381
5382 Results.push_back(
5383 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5384 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5385
5386 Results.push_back(Tmp2.getValue(1));
5387 break;
5388 }
5389 case ISD::FFLOOR:
5390 case ISD::FCEIL:
5391 case ISD::FRINT:
5392 case ISD::FNEARBYINT:
5393 case ISD::FROUND:
5394 case ISD::FROUNDEVEN:
5395 case ISD::FTRUNC:
5396 case ISD::FNEG:
5397 case ISD::FSQRT:
5398 case ISD::FSIN:
5399 case ISD::FCOS:
5400 case ISD::FLOG:
5401 case ISD::FLOG2:
5402 case ISD::FLOG10:
5403 case ISD::FABS:
5404 case ISD::FEXP:
5405 case ISD::FEXP2:
5406 case ISD::FEXP10:
5407 case ISD::FCANONICALIZE:
5408 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5409 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5410 Results.push_back(
5411 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5412 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5413 break;
5414 case ISD::STRICT_FFLOOR:
5415 case ISD::STRICT_FCEIL:
5416 case ISD::STRICT_FRINT:
5418 case ISD::STRICT_FROUND:
5420 case ISD::STRICT_FTRUNC:
5421 case ISD::STRICT_FSQRT:
5422 case ISD::STRICT_FSIN:
5423 case ISD::STRICT_FCOS:
5424 case ISD::STRICT_FLOG:
5425 case ISD::STRICT_FLOG2:
5426 case ISD::STRICT_FLOG10:
5427 case ISD::STRICT_FEXP:
5428 case ISD::STRICT_FEXP2:
5429 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5430 {Node->getOperand(0), Node->getOperand(1)});
5431 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5432 {Tmp1.getValue(1), Tmp1});
5433 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5434 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5435 Results.push_back(Tmp3);
5436 Results.push_back(Tmp3.getValue(1));
5437 break;
5438 case ISD::BUILD_VECTOR: {
5439 MVT EltVT = OVT.getVectorElementType();
5440 MVT NewEltVT = NVT.getVectorElementType();
5441
5442 // Handle bitcasts to a different vector type with the same total bit size
5443 //
5444 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5445 // =>
5446 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5447
5448 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5449 "Invalid promote type for build_vector");
5450 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5451
5452 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5453
5455 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
5456 SDValue Op = Node->getOperand(I);
5457 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
5458 }
5459
5460 SDLoc SL(Node);
5461 SDValue Concat =
5462 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5463 SL, NVT, NewOps);
5464 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5465 Results.push_back(CvtVec);
5466 break;
5467 }
5469 MVT EltVT = OVT.getVectorElementType();
5470 MVT NewEltVT = NVT.getVectorElementType();
5471
5472 // Handle bitcasts to a different vector type with the same total bit size.
5473 //
5474 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5475 // =>
5476 // v4i32:castx = bitcast x:v2i64
5477 //
5478 // i64 = bitcast
5479 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5480 // (i32 (extract_vector_elt castx, (2 * y + 1)))
5481 //
5482
5483 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5484 "Invalid promote type for extract_vector_elt");
5485 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5486
5487 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5488 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5489
5490 SDValue Idx = Node->getOperand(1);
5491 EVT IdxVT = Idx.getValueType();
5492 SDLoc SL(Node);
5493 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5494 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5495
5496 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5497
5499 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5500 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5501 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5502
5503 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5504 CastVec, TmpIdx);
5505 NewOps.push_back(Elt);
5506 }
5507
5508 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5509 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
5510 break;
5511 }
5513 MVT EltVT = OVT.getVectorElementType();
5514 MVT NewEltVT = NVT.getVectorElementType();
5515
5516 // Handle bitcasts to a different vector type with the same total bit size
5517 //
5518 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5519 // =>
5520 // v4i32:castx = bitcast x:v2i64
5521 // v2i32:casty = bitcast y:i64
5522 //
5523 // v2i64 = bitcast
5524 // (v4i32 insert_vector_elt
5525 // (v4i32 insert_vector_elt v4i32:castx,
5526 // (extract_vector_elt casty, 0), 2 * z),
5527 // (extract_vector_elt casty, 1), (2 * z + 1))
5528
5529 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5530 "Invalid promote type for insert_vector_elt");
5531 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5532
5533 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5534 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5535
5536 SDValue Val = Node->getOperand(1);
5537 SDValue Idx = Node->getOperand(2);
5538 EVT IdxVT = Idx.getValueType();
5539 SDLoc SL(Node);
5540
5541 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
5542 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5543
5544 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5545 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5546
5547 SDValue NewVec = CastVec;
5548 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5549 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5550 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5551
5552 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5553 CastVal, IdxOffset);
5554
5555 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
5556 NewVec, Elt, InEltIdx);
5557 }
5558
5559 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
5560 break;
5561 }
5562 case ISD::SCALAR_TO_VECTOR: {
5563 MVT EltVT = OVT.getVectorElementType();
5564 MVT NewEltVT = NVT.getVectorElementType();
5565
5566 // Handle bitcasts to different vector type with the same total bit size.
5567 //
5568 // e.g. v2i64 = scalar_to_vector x:i64
5569 // =>
5570 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5571 //
5572
5573 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5574 SDValue Val = Node->getOperand(0);
5575 SDLoc SL(Node);
5576
5577 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5578 SDValue Undef = DAG.getUNDEF(MidVT);
5579
5581 NewElts.push_back(CastVal);
5582 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5583 NewElts.push_back(Undef);
5584
5585 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
5586 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5587 Results.push_back(CvtVec);
5588 break;
5589 }
5590 case ISD::ATOMIC_SWAP: {
5591 AtomicSDNode *AM = cast<AtomicSDNode>(Node);
5592 SDLoc SL(Node);
5593 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
5594 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5595 "unexpected promotion type");
5596 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5597 "unexpected atomic_swap with illegal type");
5598
5599 SDValue NewAtomic
5600 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5601 DAG.getVTList(NVT, MVT::Other),
5602 { AM->getChain(), AM->getBasePtr(), CastVal },
5603 AM->getMemOperand());
5604 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
5605 Results.push_back(NewAtomic.getValue(1));
5606 break;
5607 }
5608 case ISD::SPLAT_VECTOR: {
5609 SDValue Scalar = Node->getOperand(0);
5610 MVT ScalarType = Scalar.getSimpleValueType();
5611 MVT NewScalarType = NVT.getVectorElementType();
5612 if (ScalarType.isInteger()) {
5613 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar);
5614 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5615 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5616 break;
5617 }
5618 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar);
5619 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5620 Results.push_back(
5621 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5622 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5623 break;
5624 }
5625 }
5626
5627 // Replace the original node with the legalized result.
5628 if (!Results.empty()) {
5629 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5630 ReplaceNode(Node, Results.data());
5631 } else
5632 LLVM_DEBUG(dbgs() << "Could not promote node\n");
5633}
5634
5635/// This is the entry point for the file.
5638
5639 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5640 // Use a delete listener to remove nodes which were deleted during
5641 // legalization from LegalizeNodes. This is needed to handle the situation
5642 // where a new node is allocated by the object pool to the same address of a
5643 // previously deleted node.
5644 DAGNodeDeletedListener DeleteListener(
5645 *this,
5646 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
5647
5648 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5649
5650 // Visit all the nodes. We start in topological order, so that we see
5651 // nodes with their original operands intact. Legalization can produce
5652 // new nodes which may themselves need to be legalized. Iterate until all
5653 // nodes have been legalized.
5654 while (true) {
5655 bool AnyLegalized = false;
5656 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5657 --NI;
5658
5659 SDNode *N = &*NI;
5660 if (N->use_empty() && N != getRoot().getNode()) {
5661 ++NI;
5662 DeleteNode(N);
5663 continue;
5664 }
5665
5666 if (LegalizedNodes.insert(N).second) {
5667 AnyLegalized = true;
5668 Legalizer.LegalizeOp(N);
5669
5670 if (N->use_empty() && N != getRoot().getNode()) {
5671 ++NI;
5672 DeleteNode(N);
5673 }
5674 }
5675 }
5676 if (!AnyLegalized)
5677 break;
5678
5679 }
5680
5681 // Remove dead nodes now.
5683}
5684
5686 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5687 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5688 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5689
5690 // Directly insert the node in question, and legalize it. This will recurse
5691 // as needed through operands.
5692 LegalizedNodes.insert(N);
5693 Legalizer.LegalizeOp(N);
5694
5695 return LegalizedNodes.count(N);
5696}
#define Success
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:301
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
Definition: IRBuilder.cpp:530
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
This file contains the declarations for metadata subclasses.
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
DEMANGLE_DUMP_METHOD void dump() const
bool isSignaling() const
Definition: APFloat.h:1297
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition: APFloat.h:1026
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1308
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition: APInt.h:236
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:267
const APFloat & getValueAPF() const
Definition: Constants.h:310
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:153
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1398
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
bool isBigEndian() const
Definition: DataLayout.h:239
bool empty() const
Definition: Function.h:804
const BasicBlock & back() const
Definition: Function.h:807
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:531
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:532
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_type size() const
Definition: SmallSet.h:161
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void reserve(size_type N)
Definition: SmallVector.h:676
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
LLVM Value Representation.
Definition: Value.h:74
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1126
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1122
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:998
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:147
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1155
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
Definition: ISDOpcodes.h:124
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1002
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1021
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:151
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1233
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1025
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1199
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1400
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1102
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:135
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1188
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:101
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:988
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:758
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1056
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1151
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:880
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1016
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1266
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ TargetConstantFP
Definition: ISDOpcodes.h:159
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1211
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1248
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1215
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1041
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
@ ConstantPool
Definition: ISDOpcodes.h:82
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
Definition: ISDOpcodes.h:87
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:924
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:129
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1182
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:158
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1208
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1007
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:925
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:106
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:141
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1070
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
Definition: ISDOpcodes.h:1140
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1320
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1012
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1205
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:1556
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
constexpr double e
Definition: MathExtras.h:31
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:269
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Definition: APFloat.h:1361
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
Definition: APFloat.h:147
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition: ValueTypes.h:120
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
Definition: ValueTypes.h:415
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition: ValueTypes.h:397
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:246
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)