LLVM  16.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/IR/CallingConv.h"
34 #include "llvm/IR/Constants.h"
35 #include "llvm/IR/DataLayout.h"
36 #include "llvm/IR/DerivedTypes.h"
37 #include "llvm/IR/Function.h"
38 #include "llvm/IR/Metadata.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/Support/Casting.h"
41 #include "llvm/Support/Compiler.h"
42 #include "llvm/Support/Debug.h"
49 #include <cassert>
50 #include <cstdint>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "legalizedag"
57 
58 namespace {
59 
60 /// Keeps track of state when getting the sign of a floating-point value as an
61 /// integer.
62 struct FloatSignAsInt {
63  EVT FloatVT;
64  SDValue Chain;
65  SDValue FloatPtr;
66  SDValue IntPtr;
67  MachinePointerInfo IntPointerInfo;
68  MachinePointerInfo FloatPointerInfo;
69  SDValue IntValue;
70  APInt SignMask;
71  uint8_t SignBit;
72 };
73 
74 //===----------------------------------------------------------------------===//
75 /// This takes an arbitrary SelectionDAG as input and
76 /// hacks on it until the target machine can handle it. This involves
77 /// eliminating value sizes the machine cannot handle (promoting small sizes to
78 /// large sizes or splitting up large values into small values) as well as
79 /// eliminating operations the machine cannot handle.
80 ///
81 /// This code also does a small amount of optimization and recognition of idioms
82 /// as part of its processing. For example, if a target does not support a
83 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84 /// will attempt merge setcc and brc instructions into brcc's.
85 class SelectionDAGLegalize {
86  const TargetMachine &TM;
87  const TargetLowering &TLI;
88  SelectionDAG &DAG;
89 
90  /// The set of nodes which have already been legalized. We hold a
91  /// reference to it in order to update as necessary on node deletion.
92  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 
94  /// A set of all the nodes updated during legalization.
95  SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 
97  EVT getSetCCResultType(EVT VT) const {
98  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99  }
100 
101  // Libcall insertion helpers.
102 
103 public:
104  SelectionDAGLegalize(SelectionDAG &DAG,
105  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 
110  /// Legalizes the given operation.
111  void LegalizeOp(SDNode *Node);
112 
113 private:
114  SDValue OptimizeFloatStore(StoreSDNode *ST);
115 
116  void LegalizeLoadOps(SDNode *Node);
117  void LegalizeStoreOps(SDNode *Node);
118 
119  /// Some targets cannot handle a variable
120  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121  /// is necessary to spill the vector being inserted into to memory, perform
122  /// the insert there, and then read the result back.
123  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124  const SDLoc &dl);
125  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126  const SDLoc &dl);
127 
128  /// Return a vector shuffle operation which
129  /// performs the same shuffe in terms of order or result bytes, but on a type
130  /// whose vector element type is narrower than the original shuffle type.
131  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133  SDValue N1, SDValue N2,
134  ArrayRef<int> Mask) const;
135 
136  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
137 
138  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
140  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
141  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
142  RTLIB::Libcall Call_F128,
143  RTLIB::Libcall Call_PPCF128,
145  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
146  RTLIB::Libcall Call_I8,
147  RTLIB::Libcall Call_I16,
148  RTLIB::Libcall Call_I32,
149  RTLIB::Libcall Call_I64,
150  RTLIB::Libcall Call_I128);
151  void ExpandArgFPLibCall(SDNode *Node,
152  RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
153  RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
154  RTLIB::Libcall Call_PPCF128,
156  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158 
159  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160  const SDLoc &dl);
161  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162  const SDLoc &dl, SDValue ChainIn);
163  SDValue ExpandBUILD_VECTOR(SDNode *Node);
164  SDValue ExpandSPLAT_VECTOR(SDNode *Node);
165  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
166  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
168  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
169  SDValue Value) const;
170  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
171  SDValue NewIntValue) const;
172  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
173  SDValue ExpandFABS(SDNode *Node) const;
174  SDValue ExpandFNEG(SDNode *Node) const;
175  SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
176  void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
178  void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
180  SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
181 
182  SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
183 
184  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
185  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
186  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
187 
188  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
189  SDValue ExpandConstant(ConstantSDNode *CP);
190 
191  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
192  bool ExpandNode(SDNode *Node);
193  void ConvertNodeToLibcall(SDNode *Node);
194  void PromoteNode(SDNode *Node);
195 
196 public:
197  // Node replacement helpers
198 
199  void ReplacedNode(SDNode *N) {
200  LegalizedNodes.erase(N);
201  if (UpdatedNodes)
202  UpdatedNodes->insert(N);
203  }
204 
205  void ReplaceNode(SDNode *Old, SDNode *New) {
206  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
207  dbgs() << " with: "; New->dump(&DAG));
208 
209  assert(Old->getNumValues() == New->getNumValues() &&
210  "Replacing one node with another that produces a different number "
211  "of values!");
212  DAG.ReplaceAllUsesWith(Old, New);
213  if (UpdatedNodes)
214  UpdatedNodes->insert(New);
215  ReplacedNode(Old);
216  }
217 
218  void ReplaceNode(SDValue Old, SDValue New) {
219  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
220  dbgs() << " with: "; New->dump(&DAG));
221 
222  DAG.ReplaceAllUsesWith(Old, New);
223  if (UpdatedNodes)
224  UpdatedNodes->insert(New.getNode());
225  ReplacedNode(Old.getNode());
226  }
227 
228  void ReplaceNode(SDNode *Old, const SDValue *New) {
229  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
230 
231  DAG.ReplaceAllUsesWith(Old, New);
232  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
233  LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
234  New[i]->dump(&DAG));
235  if (UpdatedNodes)
236  UpdatedNodes->insert(New[i].getNode());
237  }
238  ReplacedNode(Old);
239  }
240 
241  void ReplaceNodeWithValue(SDValue Old, SDValue New) {
242  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
243  dbgs() << " with: "; New->dump(&DAG));
244 
245  DAG.ReplaceAllUsesOfValueWith(Old, New);
246  if (UpdatedNodes)
247  UpdatedNodes->insert(New.getNode());
248  ReplacedNode(Old.getNode());
249  }
250 };
251 
252 } // end anonymous namespace
253 
254 /// Return a vector shuffle operation which
255 /// performs the same shuffle in terms of order or result bytes, but on a type
256 /// whose vector element type is narrower than the original shuffle type.
257 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
258 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
259  EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
260  ArrayRef<int> Mask) const {
261  unsigned NumMaskElts = VT.getVectorNumElements();
262  unsigned NumDestElts = NVT.getVectorNumElements();
263  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
264 
265  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
266 
267  if (NumEltsGrowth == 1)
268  return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
269 
270  SmallVector<int, 8> NewMask;
271  for (unsigned i = 0; i != NumMaskElts; ++i) {
272  int Idx = Mask[i];
273  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
274  if (Idx < 0)
275  NewMask.push_back(-1);
276  else
277  NewMask.push_back(Idx * NumEltsGrowth + j);
278  }
279  }
280  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
281  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
282  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
283 }
284 
285 /// Expands the ConstantFP node to an integer constant or
286 /// a load from the constant pool.
287 SDValue
288 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
289  bool Extend = false;
290  SDLoc dl(CFP);
291 
292  // If a FP immediate is precise when represented as a float and if the
293  // target can do an extending load from float to double, we put it into
294  // the constant pool as a float, even if it's is statically typed as a
295  // double. This shrinks FP constants and canonicalizes them for targets where
296  // an FP extending load is the same cost as a normal load (such as on the x87
297  // fp stack or PPC FP unit).
298  EVT VT = CFP->getValueType(0);
299  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
300  if (!UseCP) {
301  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
302  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
303  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
304  }
305 
306  APFloat APF = CFP->getValueAPF();
307  EVT OrigVT = VT;
308  EVT SVT = VT;
309 
310  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
311  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
312  if (!APF.isSignaling()) {
313  while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
314  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
316  // Only do this if the target has a native EXTLOAD instruction from
317  // smaller type.
318  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
319  TLI.ShouldShrinkFPConstant(OrigVT)) {
320  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
321  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
322  VT = SVT;
323  Extend = true;
324  }
325  }
326  }
327 
328  SDValue CPIdx =
329  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
330  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
331  if (Extend) {
332  SDValue Result = DAG.getExtLoad(
333  ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
334  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
335  Alignment);
336  return Result;
337  }
338  SDValue Result = DAG.getLoad(
339  OrigVT, dl, DAG.getEntryNode(), CPIdx,
340  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
341  return Result;
342 }
343 
344 /// Expands the Constant node to a load from the constant pool.
345 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
346  SDLoc dl(CP);
347  EVT VT = CP->getValueType(0);
348  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
349  TLI.getPointerTy(DAG.getDataLayout()));
350  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
351  SDValue Result = DAG.getLoad(
352  VT, dl, DAG.getEntryNode(), CPIdx,
353  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
354  return Result;
355 }
356 
357 /// Some target cannot handle a variable insertion index for the
358 /// INSERT_VECTOR_ELT instruction. In this case, it
359 /// is necessary to spill the vector being inserted into to memory, perform
360 /// the insert there, and then read the result back.
361 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
362  SDValue Val,
363  SDValue Idx,
364  const SDLoc &dl) {
365  SDValue Tmp1 = Vec;
366  SDValue Tmp2 = Val;
367  SDValue Tmp3 = Idx;
368 
369  // If the target doesn't support this, we have to spill the input vector
370  // to a temporary stack slot, update the element, then reload it. This is
371  // badness. We could also load the value into a vector register (either
372  // with a "move to register" or "extload into register" instruction, then
373  // permute it into place, if the idx is a constant and if the idx is
374  // supported by the target.
375  EVT VT = Tmp1.getValueType();
376  EVT EltVT = VT.getVectorElementType();
377  SDValue StackPtr = DAG.CreateStackTemporary(VT);
378 
379  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
380 
381  // Store the vector.
382  SDValue Ch = DAG.getStore(
383  DAG.getEntryNode(), dl, Tmp1, StackPtr,
384  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
385 
386  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
387 
388  // Store the scalar value.
389  Ch = DAG.getTruncStore(
390  Ch, dl, Tmp2, StackPtr2,
391  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
392  // Load the updated vector.
393  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
394  DAG.getMachineFunction(), SPFI));
395 }
396 
397 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
398  SDValue Idx,
399  const SDLoc &dl) {
400  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
401  // SCALAR_TO_VECTOR requires that the type of the value being inserted
402  // match the element type of the vector being created, except for
403  // integers in which case the inserted value can be over width.
404  EVT EltVT = Vec.getValueType().getVectorElementType();
405  if (Val.getValueType() == EltVT ||
406  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
407  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
408  Vec.getValueType(), Val);
409 
410  unsigned NumElts = Vec.getValueType().getVectorNumElements();
411  // We generate a shuffle of InVec and ScVec, so the shuffle mask
412  // should be 0,1,2,3,4,5... with the appropriate element replaced with
413  // elt 0 of the RHS.
414  SmallVector<int, 8> ShufOps;
415  for (unsigned i = 0; i != NumElts; ++i)
416  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
417 
418  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
419  }
420  }
421  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
422 }
423 
424 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
425  if (!ISD::isNormalStore(ST))
426  return SDValue();
427 
428  LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
429  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
430  // FIXME: move this to the DAG Combiner! Note that we can't regress due
431  // to phase ordering between legalized code and the dag combiner. This
432  // probably means that we need to integrate dag combiner and legalizer
433  // together.
434  // We generally can't do this one for long doubles.
435  SDValue Chain = ST->getChain();
436  SDValue Ptr = ST->getBasePtr();
437  SDValue Value = ST->getValue();
438  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
439  AAMDNodes AAInfo = ST->getAAInfo();
440  SDLoc dl(ST);
441 
442  // Don't optimise TargetConstantFP
443  if (Value.getOpcode() == ISD::TargetConstantFP)
444  return SDValue();
445 
446  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
447  if (CFP->getValueType(0) == MVT::f32 &&
448  TLI.isTypeLegal(MVT::i32)) {
449  SDValue Con = DAG.getConstant(CFP->getValueAPF().
450  bitcastToAPInt().zextOrTrunc(32),
451  SDLoc(CFP), MVT::i32);
452  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
453  ST->getOriginalAlign(), MMOFlags, AAInfo);
454  }
455 
456  if (CFP->getValueType(0) == MVT::f64) {
457  // If this target supports 64-bit registers, do a single 64-bit store.
458  if (TLI.isTypeLegal(MVT::i64)) {
459  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
460  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
461  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
462  ST->getOriginalAlign(), MMOFlags, AAInfo);
463  }
464 
465  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
466  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
467  // stores. If the target supports neither 32- nor 64-bits, this
468  // xform is certainly not worth it.
469  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
470  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
471  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
472  if (DAG.getDataLayout().isBigEndian())
473  std::swap(Lo, Hi);
474 
475  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
476  ST->getOriginalAlign(), MMOFlags, AAInfo);
477  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl);
478  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
479  ST->getPointerInfo().getWithOffset(4),
480  ST->getOriginalAlign(), MMOFlags, AAInfo);
481 
482  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
483  }
484  }
485  }
486  return SDValue();
487 }
488 
489 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
490  StoreSDNode *ST = cast<StoreSDNode>(Node);
491  SDValue Chain = ST->getChain();
492  SDValue Ptr = ST->getBasePtr();
493  SDLoc dl(Node);
494 
495  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
496  AAMDNodes AAInfo = ST->getAAInfo();
497 
498  if (!ST->isTruncatingStore()) {
499  LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
500  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
501  ReplaceNode(ST, OptStore);
502  return;
503  }
504 
505  SDValue Value = ST->getValue();
506  MVT VT = Value.getSimpleValueType();
507  switch (TLI.getOperationAction(ISD::STORE, VT)) {
508  default: llvm_unreachable("This action is not supported yet!");
509  case TargetLowering::Legal: {
510  // If this is an unaligned store and the target doesn't support it,
511  // expand it.
512  EVT MemVT = ST->getMemoryVT();
513  const DataLayout &DL = DAG.getDataLayout();
514  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
515  *ST->getMemOperand())) {
516  LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
517  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
518  ReplaceNode(SDValue(ST, 0), Result);
519  } else
520  LLVM_DEBUG(dbgs() << "Legal store\n");
521  break;
522  }
523  case TargetLowering::Custom: {
524  LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
525  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
526  if (Res && Res != SDValue(Node, 0))
527  ReplaceNode(SDValue(Node, 0), Res);
528  return;
529  }
531  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
532  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
533  "Can only promote stores to same size type");
534  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
535  SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
536  ST->getOriginalAlign(), MMOFlags, AAInfo);
537  ReplaceNode(SDValue(Node, 0), Result);
538  break;
539  }
540  }
541  return;
542  }
543 
544  LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
545  SDValue Value = ST->getValue();
546  EVT StVT = ST->getMemoryVT();
547  TypeSize StWidth = StVT.getSizeInBits();
548  TypeSize StSize = StVT.getStoreSizeInBits();
549  auto &DL = DAG.getDataLayout();
550 
551  if (StWidth != StSize) {
552  // Promote to a byte-sized store with upper bits zero if not
553  // storing an integral number of bytes. For example, promote
554  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
555  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize());
556  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
557  SDValue Result =
558  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
559  ST->getOriginalAlign(), MMOFlags, AAInfo);
560  ReplaceNode(SDValue(Node, 0), Result);
561  } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) {
562  // If not storing a power-of-2 number of bits, expand as two stores.
563  assert(!StVT.isVector() && "Unsupported truncstore!");
564  unsigned StWidthBits = StWidth.getFixedSize();
565  unsigned LogStWidth = Log2_32(StWidthBits);
566  assert(LogStWidth < 32);
567  unsigned RoundWidth = 1 << LogStWidth;
568  assert(RoundWidth < StWidthBits);
569  unsigned ExtraWidth = StWidthBits - RoundWidth;
570  assert(ExtraWidth < RoundWidth);
571  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
572  "Store size not an integral number of bytes!");
573  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
574  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
575  SDValue Lo, Hi;
576  unsigned IncrementSize;
577 
578  if (DL.isLittleEndian()) {
579  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
580  // Store the bottom RoundWidth bits.
581  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
582  RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
583 
584  // Store the remaining ExtraWidth bits.
585  IncrementSize = RoundWidth / 8;
586  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
587  Hi = DAG.getNode(
588  ISD::SRL, dl, Value.getValueType(), Value,
589  DAG.getConstant(RoundWidth, dl,
590  TLI.getShiftAmountTy(Value.getValueType(), DL)));
591  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
592  ST->getPointerInfo().getWithOffset(IncrementSize),
593  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
594  } else {
595  // Big endian - avoid unaligned stores.
596  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
597  // Store the top RoundWidth bits.
598  Hi = DAG.getNode(
599  ISD::SRL, dl, Value.getValueType(), Value,
600  DAG.getConstant(ExtraWidth, dl,
601  TLI.getShiftAmountTy(Value.getValueType(), DL)));
602  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
603  ST->getOriginalAlign(), MMOFlags, AAInfo);
604 
605  // Store the remaining ExtraWidth bits.
606  IncrementSize = RoundWidth / 8;
607  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
608  DAG.getConstant(IncrementSize, dl,
609  Ptr.getValueType()));
610  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
611  ST->getPointerInfo().getWithOffset(IncrementSize),
612  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
613  }
614 
615  // The order of the stores doesn't matter.
616  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
617  ReplaceNode(SDValue(Node, 0), Result);
618  } else {
619  switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
620  default: llvm_unreachable("This action is not supported yet!");
621  case TargetLowering::Legal: {
622  EVT MemVT = ST->getMemoryVT();
623  // If this is an unaligned store and the target doesn't support it,
624  // expand it.
625  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
626  *ST->getMemOperand())) {
627  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
628  ReplaceNode(SDValue(ST, 0), Result);
629  }
630  break;
631  }
632  case TargetLowering::Custom: {
633  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
634  if (Res && Res != SDValue(Node, 0))
635  ReplaceNode(SDValue(Node, 0), Res);
636  return;
637  }
639  assert(!StVT.isVector() &&
640  "Vector Stores are handled in LegalizeVectorOps");
641 
642  SDValue Result;
643 
644  // TRUNCSTORE:i16 i32 -> STORE i16
645  if (TLI.isTypeLegal(StVT)) {
646  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
647  Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
648  ST->getOriginalAlign(), MMOFlags, AAInfo);
649  } else {
650  // The in-memory type isn't legal. Truncate to the type it would promote
651  // to, and then do a truncstore.
652  Value = DAG.getNode(ISD::TRUNCATE, dl,
653  TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
654  Value);
655  Result =
656  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
657  ST->getOriginalAlign(), MMOFlags, AAInfo);
658  }
659 
660  ReplaceNode(SDValue(Node, 0), Result);
661  break;
662  }
663  }
664 }
665 
666 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
667  LoadSDNode *LD = cast<LoadSDNode>(Node);
668  SDValue Chain = LD->getChain(); // The chain.
669  SDValue Ptr = LD->getBasePtr(); // The base pointer.
670  SDValue Value; // The value returned by the load op.
671  SDLoc dl(Node);
672 
673  ISD::LoadExtType ExtType = LD->getExtensionType();
674  if (ExtType == ISD::NON_EXTLOAD) {
675  LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
676  MVT VT = Node->getSimpleValueType(0);
677  SDValue RVal = SDValue(Node, 0);
678  SDValue RChain = SDValue(Node, 1);
679 
680  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
681  default: llvm_unreachable("This action is not supported yet!");
682  case TargetLowering::Legal: {
683  EVT MemVT = LD->getMemoryVT();
684  const DataLayout &DL = DAG.getDataLayout();
685  // If this is an unaligned load and the target doesn't support it,
686  // expand it.
687  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
688  *LD->getMemOperand())) {
689  std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
690  }
691  break;
692  }
694  if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
695  RVal = Res;
696  RChain = Res.getValue(1);
697  }
698  break;
699 
701  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
702  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
703  "Can only promote loads to same size type");
704 
705  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
706  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
707  RChain = Res.getValue(1);
708  break;
709  }
710  }
711  if (RChain.getNode() != Node) {
712  assert(RVal.getNode() != Node && "Load must be completely replaced");
713  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
714  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
715  if (UpdatedNodes) {
716  UpdatedNodes->insert(RVal.getNode());
717  UpdatedNodes->insert(RChain.getNode());
718  }
719  ReplacedNode(Node);
720  }
721  return;
722  }
723 
724  LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
725  EVT SrcVT = LD->getMemoryVT();
726  TypeSize SrcWidth = SrcVT.getSizeInBits();
727  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
728  AAMDNodes AAInfo = LD->getAAInfo();
729 
730  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
731  // Some targets pretend to have an i1 loading operation, and actually
732  // load an i8. This trick is correct for ZEXTLOAD because the top 7
733  // bits are guaranteed to be zero; it helps the optimizers understand
734  // that these bits are zero. It is also useful for EXTLOAD, since it
735  // tells the optimizers that those bits are undefined. It would be
736  // nice to have an effective generic way of getting these benefits...
737  // Until such a way is found, don't insist on promoting i1 here.
738  (SrcVT != MVT::i1 ||
739  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
741  // Promote to a byte-sized load if not loading an integral number of
742  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
743  unsigned NewWidth = SrcVT.getStoreSizeInBits();
744  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
745  SDValue Ch;
746 
747  // The extra bits are guaranteed to be zero, since we stored them that
748  // way. A zext load from NVT thus automatically gives zext from SrcVT.
749 
750  ISD::LoadExtType NewExtType =
752 
753  SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
754  Chain, Ptr, LD->getPointerInfo(), NVT,
755  LD->getOriginalAlign(), MMOFlags, AAInfo);
756 
757  Ch = Result.getValue(1); // The chain.
758 
759  if (ExtType == ISD::SEXTLOAD)
760  // Having the top bits zero doesn't help when sign extending.
761  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
762  Result.getValueType(),
763  Result, DAG.getValueType(SrcVT));
764  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
765  // All the top bits are guaranteed to be zero - inform the optimizers.
766  Result = DAG.getNode(ISD::AssertZext, dl,
767  Result.getValueType(), Result,
768  DAG.getValueType(SrcVT));
769 
770  Value = Result;
771  Chain = Ch;
772  } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) {
773  // If not loading a power-of-2 number of bits, expand as two loads.
774  assert(!SrcVT.isVector() && "Unsupported extload!");
775  unsigned SrcWidthBits = SrcWidth.getFixedSize();
776  unsigned LogSrcWidth = Log2_32(SrcWidthBits);
777  assert(LogSrcWidth < 32);
778  unsigned RoundWidth = 1 << LogSrcWidth;
779  assert(RoundWidth < SrcWidthBits);
780  unsigned ExtraWidth = SrcWidthBits - RoundWidth;
781  assert(ExtraWidth < RoundWidth);
782  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
783  "Load size not an integral number of bytes!");
784  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
785  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
786  SDValue Lo, Hi, Ch;
787  unsigned IncrementSize;
788  auto &DL = DAG.getDataLayout();
789 
790  if (DL.isLittleEndian()) {
791  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
792  // Load the bottom RoundWidth bits.
793  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
794  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
795  MMOFlags, AAInfo);
796 
797  // Load the remaining ExtraWidth bits.
798  IncrementSize = RoundWidth / 8;
799  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
800  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
801  LD->getPointerInfo().getWithOffset(IncrementSize),
802  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
803 
804  // Build a factor node to remember that this load is independent of
805  // the other one.
806  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
807  Hi.getValue(1));
808 
809  // Move the top bits to the right place.
810  Hi = DAG.getNode(
811  ISD::SHL, dl, Hi.getValueType(), Hi,
812  DAG.getConstant(RoundWidth, dl,
813  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
814 
815  // Join the hi and lo parts.
816  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
817  } else {
818  // Big endian - avoid unaligned loads.
819  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
820  // Load the top RoundWidth bits.
821  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
822  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
823  MMOFlags, AAInfo);
824 
825  // Load the remaining ExtraWidth bits.
826  IncrementSize = RoundWidth / 8;
827  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
828  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
829  LD->getPointerInfo().getWithOffset(IncrementSize),
830  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
831 
832  // Build a factor node to remember that this load is independent of
833  // the other one.
834  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
835  Hi.getValue(1));
836 
837  // Move the top bits to the right place.
838  Hi = DAG.getNode(
839  ISD::SHL, dl, Hi.getValueType(), Hi,
840  DAG.getConstant(ExtraWidth, dl,
841  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
842 
843  // Join the hi and lo parts.
844  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
845  }
846 
847  Chain = Ch;
848  } else {
849  bool isCustom = false;
850  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
851  SrcVT.getSimpleVT())) {
852  default: llvm_unreachable("This action is not supported yet!");
854  isCustom = true;
855  [[fallthrough]];
857  Value = SDValue(Node, 0);
858  Chain = SDValue(Node, 1);
859 
860  if (isCustom) {
861  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
862  Value = Res;
863  Chain = Res.getValue(1);
864  }
865  } else {
866  // If this is an unaligned load and the target doesn't support it,
867  // expand it.
868  EVT MemVT = LD->getMemoryVT();
869  const DataLayout &DL = DAG.getDataLayout();
870  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
871  *LD->getMemOperand())) {
872  std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
873  }
874  }
875  break;
876 
877  case TargetLowering::Expand: {
878  EVT DestVT = Node->getValueType(0);
879  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
880  // If the source type is not legal, see if there is a legal extload to
881  // an intermediate type that we can then extend further.
882  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
883  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
884  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
885  // If we are loading a legal type, this is a non-extload followed by a
886  // full extend.
887  ISD::LoadExtType MidExtType =
888  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
889 
890  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
891  SrcVT, LD->getMemOperand());
892  unsigned ExtendOp =
893  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
894  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
895  Chain = Load.getValue(1);
896  break;
897  }
898 
899  // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
900  // normal undefined upper bits behavior to allow using an in-reg extend
901  // with the illegal FP type, so load as an integer and do the
902  // from-integer conversion.
903  if (SrcVT.getScalarType() == MVT::f16) {
904  EVT ISrcVT = SrcVT.changeTypeToInteger();
905  EVT IDestVT = DestVT.changeTypeToInteger();
906  EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
907 
908  SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
909  Ptr, ISrcVT, LD->getMemOperand());
910  Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
911  Chain = Result.getValue(1);
912  break;
913  }
914  }
915 
916  assert(!SrcVT.isVector() &&
917  "Vector Loads are handled in LegalizeVectorOps");
918 
919  // FIXME: This does not work for vectors on most targets. Sign-
920  // and zero-extend operations are currently folded into extending
921  // loads, whether they are legal or not, and then we end up here
922  // without any support for legalizing them.
923  assert(ExtType != ISD::EXTLOAD &&
924  "EXTLOAD should always be supported!");
925  // Turn the unsupported load into an EXTLOAD followed by an
926  // explicit zero/sign extend inreg.
927  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
928  Node->getValueType(0),
929  Chain, Ptr, SrcVT,
930  LD->getMemOperand());
931  SDValue ValRes;
932  if (ExtType == ISD::SEXTLOAD)
933  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
934  Result.getValueType(),
935  Result, DAG.getValueType(SrcVT));
936  else
937  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
938  Value = ValRes;
939  Chain = Result.getValue(1);
940  break;
941  }
942  }
943  }
944 
945  // Since loads produce two values, make sure to remember that we legalized
946  // both of them.
947  if (Chain.getNode() != Node) {
948  assert(Value.getNode() != Node && "Load must be completely replaced");
949  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
950  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
951  if (UpdatedNodes) {
952  UpdatedNodes->insert(Value.getNode());
953  UpdatedNodes->insert(Chain.getNode());
954  }
955  ReplacedNode(Node);
956  }
957 }
958 
959 /// Return a legal replacement for the given operation, with all legal operands.
960 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
961  LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
962 
963  // Allow illegal target nodes and illegal registers.
964  if (Node->getOpcode() == ISD::TargetConstant ||
965  Node->getOpcode() == ISD::Register)
966  return;
967 
968 #ifndef NDEBUG
969  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
970  assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
972  "Unexpected illegal type!");
973 
974  for (const SDValue &Op : Node->op_values())
975  assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
977  Op.getOpcode() == ISD::TargetConstant ||
978  Op.getOpcode() == ISD::Register) &&
979  "Unexpected illegal type!");
980 #endif
981 
982  // Figure out the correct action; the way to query this varies by opcode
984  bool SimpleFinishLegalizing = true;
985  switch (Node->getOpcode()) {
988  case ISD::INTRINSIC_VOID:
989  case ISD::STACKSAVE:
990  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
991  break;
993  Action = TLI.getOperationAction(Node->getOpcode(),
994  Node->getValueType(0));
995  break;
996  case ISD::VAARG:
997  Action = TLI.getOperationAction(Node->getOpcode(),
998  Node->getValueType(0));
999  if (Action != TargetLowering::Promote)
1000  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1001  break;
1002  case ISD::FP_TO_FP16:
1003  case ISD::FP_TO_BF16:
1004  case ISD::SINT_TO_FP:
1005  case ISD::UINT_TO_FP:
1007  case ISD::LROUND:
1008  case ISD::LLROUND:
1009  case ISD::LRINT:
1010  case ISD::LLRINT:
1011  Action = TLI.getOperationAction(Node->getOpcode(),
1012  Node->getOperand(0).getValueType());
1013  break;
1017  case ISD::STRICT_LRINT:
1018  case ISD::STRICT_LLRINT:
1019  case ISD::STRICT_LROUND:
1020  case ISD::STRICT_LLROUND:
1021  // These pseudo-ops are the same as the other STRICT_ ops except
1022  // they are registered with setOperationAction() using the input type
1023  // instead of the output type.
1024  Action = TLI.getOperationAction(Node->getOpcode(),
1025  Node->getOperand(1).getValueType());
1026  break;
1027  case ISD::SIGN_EXTEND_INREG: {
1028  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1029  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1030  break;
1031  }
1032  case ISD::ATOMIC_STORE:
1033  Action = TLI.getOperationAction(Node->getOpcode(),
1034  Node->getOperand(2).getValueType());
1035  break;
1036  case ISD::SELECT_CC:
1037  case ISD::STRICT_FSETCC:
1038  case ISD::STRICT_FSETCCS:
1039  case ISD::SETCC:
1040  case ISD::SETCCCARRY:
1041  case ISD::VP_SETCC:
1042  case ISD::BR_CC: {
1043  unsigned Opc = Node->getOpcode();
1044  unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1045  : Opc == ISD::STRICT_FSETCC ? 3
1046  : Opc == ISD::STRICT_FSETCCS ? 3
1047  : Opc == ISD::SETCCCARRY ? 3
1048  : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1049  : 1;
1050  unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1051  : Opc == ISD::STRICT_FSETCC ? 1
1052  : Opc == ISD::STRICT_FSETCCS ? 1
1053  : 0;
1054  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1055  ISD::CondCode CCCode =
1056  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1057  Action = TLI.getCondCodeAction(CCCode, OpVT);
1058  if (Action == TargetLowering::Legal) {
1059  if (Node->getOpcode() == ISD::SELECT_CC)
1060  Action = TLI.getOperationAction(Node->getOpcode(),
1061  Node->getValueType(0));
1062  else
1063  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1064  }
1065  break;
1066  }
1067  case ISD::LOAD:
1068  case ISD::STORE:
1069  // FIXME: Model these properly. LOAD and STORE are complicated, and
1070  // STORE expects the unlegalized operand in some cases.
1071  SimpleFinishLegalizing = false;
1072  break;
1073  case ISD::CALLSEQ_START:
1074  case ISD::CALLSEQ_END:
1075  // FIXME: This shouldn't be necessary. These nodes have special properties
1076  // dealing with the recursive nature of legalization. Removing this
1077  // special case should be done as part of making LegalizeDAG non-recursive.
1078  SimpleFinishLegalizing = false;
1079  break;
1080  case ISD::EXTRACT_ELEMENT:
1081  case ISD::FLT_ROUNDS_:
1082  case ISD::MERGE_VALUES:
1083  case ISD::EH_RETURN:
1085  case ISD::EH_DWARF_CFA:
1086  case ISD::EH_SJLJ_SETJMP:
1087  case ISD::EH_SJLJ_LONGJMP:
1089  // These operations lie about being legal: when they claim to be legal,
1090  // they should actually be expanded.
1091  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1092  if (Action == TargetLowering::Legal)
1093  Action = TargetLowering::Expand;
1094  break;
1095  case ISD::INIT_TRAMPOLINE:
1097  case ISD::FRAMEADDR:
1098  case ISD::RETURNADDR:
1099  case ISD::ADDROFRETURNADDR:
1100  case ISD::SPONENTRY:
1101  // These operations lie about being legal: when they claim to be legal,
1102  // they should actually be custom-lowered.
1103  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1104  if (Action == TargetLowering::Legal)
1105  Action = TargetLowering::Custom;
1106  break;
1107  case ISD::READCYCLECOUNTER:
1108  // READCYCLECOUNTER returns an i64, even if type legalization might have
1109  // expanded that to several smaller types.
1110  Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1111  break;
1112  case ISD::READ_REGISTER:
1113  case ISD::WRITE_REGISTER:
1114  // Named register is legal in the DAG, but blocked by register name
1115  // selection if not implemented by target (to chose the correct register)
1116  // They'll be converted to Copy(To/From)Reg.
1117  Action = TargetLowering::Legal;
1118  break;
1119  case ISD::UBSANTRAP:
1120  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1121  if (Action == TargetLowering::Expand) {
1122  // replace ISD::UBSANTRAP with ISD::TRAP
1123  SDValue NewVal;
1124  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1125  Node->getOperand(0));
1126  ReplaceNode(Node, NewVal.getNode());
1127  LegalizeOp(NewVal.getNode());
1128  return;
1129  }
1130  break;
1131  case ISD::DEBUGTRAP:
1132  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1133  if (Action == TargetLowering::Expand) {
1134  // replace ISD::DEBUGTRAP with ISD::TRAP
1135  SDValue NewVal;
1136  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1137  Node->getOperand(0));
1138  ReplaceNode(Node, NewVal.getNode());
1139  LegalizeOp(NewVal.getNode());
1140  return;
1141  }
1142  break;
1143  case ISD::SADDSAT:
1144  case ISD::UADDSAT:
1145  case ISD::SSUBSAT:
1146  case ISD::USUBSAT:
1147  case ISD::SSHLSAT:
1148  case ISD::USHLSAT:
1149  case ISD::FP_TO_SINT_SAT:
1150  case ISD::FP_TO_UINT_SAT:
1151  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1152  break;
1153  case ISD::SMULFIX:
1154  case ISD::SMULFIXSAT:
1155  case ISD::UMULFIX:
1156  case ISD::UMULFIXSAT:
1157  case ISD::SDIVFIX:
1158  case ISD::SDIVFIXSAT:
1159  case ISD::UDIVFIX:
1160  case ISD::UDIVFIXSAT: {
1161  unsigned Scale = Node->getConstantOperandVal(2);
1162  Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1163  Node->getValueType(0), Scale);
1164  break;
1165  }
1166  case ISD::MSCATTER:
1167  Action = TLI.getOperationAction(Node->getOpcode(),
1168  cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1169  break;
1170  case ISD::MSTORE:
1171  Action = TLI.getOperationAction(Node->getOpcode(),
1172  cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1173  break;
1174  case ISD::VP_SCATTER:
1175  Action = TLI.getOperationAction(
1176  Node->getOpcode(),
1177  cast<VPScatterSDNode>(Node)->getValue().getValueType());
1178  break;
1179  case ISD::VP_STORE:
1180  Action = TLI.getOperationAction(
1181  Node->getOpcode(),
1182  cast<VPStoreSDNode>(Node)->getValue().getValueType());
1183  break;
1184  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1185  Action = TLI.getOperationAction(
1186  Node->getOpcode(),
1187  cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1188  break;
1189  case ISD::VECREDUCE_FADD:
1190  case ISD::VECREDUCE_FMUL:
1191  case ISD::VECREDUCE_ADD:
1192  case ISD::VECREDUCE_MUL:
1193  case ISD::VECREDUCE_AND:
1194  case ISD::VECREDUCE_OR:
1195  case ISD::VECREDUCE_XOR:
1196  case ISD::VECREDUCE_SMAX:
1197  case ISD::VECREDUCE_SMIN:
1198  case ISD::VECREDUCE_UMAX:
1199  case ISD::VECREDUCE_UMIN:
1200  case ISD::VECREDUCE_FMAX:
1201  case ISD::VECREDUCE_FMIN:
1202  case ISD::IS_FPCLASS:
1203  Action = TLI.getOperationAction(
1204  Node->getOpcode(), Node->getOperand(0).getValueType());
1205  break;
1208  case ISD::VP_REDUCE_FADD:
1209  case ISD::VP_REDUCE_FMUL:
1210  case ISD::VP_REDUCE_ADD:
1211  case ISD::VP_REDUCE_MUL:
1212  case ISD::VP_REDUCE_AND:
1213  case ISD::VP_REDUCE_OR:
1214  case ISD::VP_REDUCE_XOR:
1215  case ISD::VP_REDUCE_SMAX:
1216  case ISD::VP_REDUCE_SMIN:
1217  case ISD::VP_REDUCE_UMAX:
1218  case ISD::VP_REDUCE_UMIN:
1219  case ISD::VP_REDUCE_FMAX:
1220  case ISD::VP_REDUCE_FMIN:
1221  case ISD::VP_REDUCE_SEQ_FADD:
1222  case ISD::VP_REDUCE_SEQ_FMUL:
1223  Action = TLI.getOperationAction(
1224  Node->getOpcode(), Node->getOperand(1).getValueType());
1225  break;
1226  default:
1227  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1228  Action = TLI.getCustomOperationAction(*Node);
1229  } else {
1230  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1231  }
1232  break;
1233  }
1234 
1235  if (SimpleFinishLegalizing) {
1236  SDNode *NewNode = Node;
1237  switch (Node->getOpcode()) {
1238  default: break;
1239  case ISD::SHL:
1240  case ISD::SRL:
1241  case ISD::SRA:
1242  case ISD::ROTL:
1243  case ISD::ROTR: {
1244  // Legalizing shifts/rotates requires adjusting the shift amount
1245  // to the appropriate width.
1246  SDValue Op0 = Node->getOperand(0);
1247  SDValue Op1 = Node->getOperand(1);
1248  if (!Op1.getValueType().isVector()) {
1249  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1250  // The getShiftAmountOperand() may create a new operand node or
1251  // return the existing one. If new operand is created we need
1252  // to update the parent node.
1253  // Do not try to legalize SAO here! It will be automatically legalized
1254  // in the next round.
1255  if (SAO != Op1)
1256  NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1257  }
1258  }
1259  break;
1260  case ISD::FSHL:
1261  case ISD::FSHR:
1262  case ISD::SRL_PARTS:
1263  case ISD::SRA_PARTS:
1264  case ISD::SHL_PARTS: {
1265  // Legalizing shifts/rotates requires adjusting the shift amount
1266  // to the appropriate width.
1267  SDValue Op0 = Node->getOperand(0);
1268  SDValue Op1 = Node->getOperand(1);
1269  SDValue Op2 = Node->getOperand(2);
1270  if (!Op2.getValueType().isVector()) {
1271  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1272  // The getShiftAmountOperand() may create a new operand node or
1273  // return the existing one. If new operand is created we need
1274  // to update the parent node.
1275  if (SAO != Op2)
1276  NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1277  }
1278  break;
1279  }
1280  }
1281 
1282  if (NewNode != Node) {
1283  ReplaceNode(Node, NewNode);
1284  Node = NewNode;
1285  }
1286  switch (Action) {
1287  case TargetLowering::Legal:
1288  LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1289  return;
1291  LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1292  // FIXME: The handling for custom lowering with multiple results is
1293  // a complete mess.
1294  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1295  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1296  return;
1297 
1298  if (Node->getNumValues() == 1) {
1299  // Verify the new types match the original. Glue is waived because
1300  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1301  assert((Res.getValueType() == Node->getValueType(0) ||
1302  Node->getValueType(0) == MVT::Glue) &&
1303  "Type mismatch for custom legalized operation");
1304  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1305  // We can just directly replace this node with the lowered value.
1306  ReplaceNode(SDValue(Node, 0), Res);
1307  return;
1308  }
1309 
1310  SmallVector<SDValue, 8> ResultVals;
1311  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1312  // Verify the new types match the original. Glue is waived because
1313  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1314  assert((Res->getValueType(i) == Node->getValueType(i) ||
1315  Node->getValueType(i) == MVT::Glue) &&
1316  "Type mismatch for custom legalized operation");
1317  ResultVals.push_back(Res.getValue(i));
1318  }
1319  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1320  ReplaceNode(Node, ResultVals.data());
1321  return;
1322  }
1323  LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1324  [[fallthrough]];
1326  if (ExpandNode(Node))
1327  return;
1328  [[fallthrough]];
1330  ConvertNodeToLibcall(Node);
1331  return;
1333  PromoteNode(Node);
1334  return;
1335  }
1336  }
1337 
1338  switch (Node->getOpcode()) {
1339  default:
1340 #ifndef NDEBUG
1341  dbgs() << "NODE: ";
1342  Node->dump( &DAG);
1343  dbgs() << "\n";
1344 #endif
1345  llvm_unreachable("Do not know how to legalize this operator!");
1346 
1347  case ISD::CALLSEQ_START:
1348  case ISD::CALLSEQ_END:
1349  break;
1350  case ISD::LOAD:
1351  return LegalizeLoadOps(Node);
1352  case ISD::STORE:
1353  return LegalizeStoreOps(Node);
1354  }
1355 }
1356 
1357 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1358  SDValue Vec = Op.getOperand(0);
1359  SDValue Idx = Op.getOperand(1);
1360  SDLoc dl(Op);
1361 
1362  // Before we generate a new store to a temporary stack slot, see if there is
1363  // already one that we can use. There often is because when we scalarize
1364  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1365  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1366  // the vector. If all are expanded here, we don't want one store per vector
1367  // element.
1368 
1369  // Caches for hasPredecessorHelper
1372  Visited.insert(Op.getNode());
1373  Worklist.push_back(Idx.getNode());
1374  SDValue StackPtr, Ch;
1375  for (SDNode *User : Vec.getNode()->uses()) {
1376  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1377  if (ST->isIndexed() || ST->isTruncatingStore() ||
1378  ST->getValue() != Vec)
1379  continue;
1380 
1381  // Make sure that nothing else could have stored into the destination of
1382  // this store.
1383  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1384  continue;
1385 
1386  // If the index is dependent on the store we will introduce a cycle when
1387  // creating the load (the load uses the index, and by replacing the chain
1388  // we will make the index dependent on the load). Also, the store might be
1389  // dependent on the extractelement and introduce a cycle when creating
1390  // the load.
1391  if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1392  ST->hasPredecessor(Op.getNode()))
1393  continue;
1394 
1395  StackPtr = ST->getBasePtr();
1396  Ch = SDValue(ST, 0);
1397  break;
1398  }
1399  }
1400 
1401  EVT VecVT = Vec.getValueType();
1402 
1403  if (!Ch.getNode()) {
1404  // Store the value to a temporary stack slot, then LOAD the returned part.
1405  StackPtr = DAG.CreateStackTemporary(VecVT);
1406  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1407  MachinePointerInfo());
1408  }
1409 
1410  SDValue NewLoad;
1411  Align ElementAlignment =
1412  std::min(cast<StoreSDNode>(Ch)->getAlign(),
1413  DAG.getDataLayout().getPrefTypeAlign(
1414  Op.getValueType().getTypeForEVT(*DAG.getContext())));
1415 
1416  if (Op.getValueType().isVector()) {
1417  StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1418  Op.getValueType(), Idx);
1419  NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1420  MachinePointerInfo(), ElementAlignment);
1421  } else {
1422  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1423  NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1425  ElementAlignment);
1426  }
1427 
1428  // Replace the chain going out of the store, by the one out of the load.
1429  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1430 
1431  // We introduced a cycle though, so update the loads operands, making sure
1432  // to use the original store's chain as an incoming chain.
1433  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1434  NewLoad->op_end());
1435  NewLoadOperands[0] = Ch;
1436  NewLoad =
1437  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1438  return NewLoad;
1439 }
1440 
1441 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1442  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1443 
1444  SDValue Vec = Op.getOperand(0);
1445  SDValue Part = Op.getOperand(1);
1446  SDValue Idx = Op.getOperand(2);
1447  SDLoc dl(Op);
1448 
1449  // Store the value to a temporary stack slot, then LOAD the returned part.
1450  EVT VecVT = Vec.getValueType();
1451  EVT SubVecVT = Part.getValueType();
1452  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1453  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1454  MachinePointerInfo PtrInfo =
1455  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1456 
1457  // First store the whole vector.
1458  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1459 
1460  // Then store the inserted part.
1461  SDValue SubStackPtr =
1462  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1463 
1464  // Store the subvector.
1465  Ch = DAG.getStore(
1466  Ch, dl, Part, SubStackPtr,
1467  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1468 
1469  // Finally, load the updated vector.
1470  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1471 }
1472 
1473 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1474  assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1475  Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1476  "Unexpected opcode!");
1477 
1478  // We can't handle this case efficiently. Allocate a sufficiently
1479  // aligned object on the stack, store each operand into it, then load
1480  // the result as a vector.
1481  // Create the stack frame object.
1482  EVT VT = Node->getValueType(0);
1483  EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1484  : Node->getOperand(0).getValueType();
1485  SDLoc dl(Node);
1486  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1487  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1488  MachinePointerInfo PtrInfo =
1489  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1490 
1491  // Emit a store of each element to the stack slot.
1492  SmallVector<SDValue, 8> Stores;
1493  unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1494  assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1495 
1496  // If the destination vector element type of a BUILD_VECTOR is narrower than
1497  // the source element type, only store the bits necessary.
1498  bool Truncate = isa<BuildVectorSDNode>(Node) &&
1499  MemVT.bitsLT(Node->getOperand(0).getValueType());
1500 
1501  // Store (in the right endianness) the elements to memory.
1502  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1503  // Ignore undef elements.
1504  if (Node->getOperand(i).isUndef()) continue;
1505 
1506  unsigned Offset = TypeByteSize*i;
1507 
1508  SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl);
1509 
1510  if (Truncate)
1511  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1512  Node->getOperand(i), Idx,
1513  PtrInfo.getWithOffset(Offset), MemVT));
1514  else
1515  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1516  Idx, PtrInfo.getWithOffset(Offset)));
1517  }
1518 
1519  SDValue StoreChain;
1520  if (!Stores.empty()) // Not all undef elements?
1521  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1522  else
1523  StoreChain = DAG.getEntryNode();
1524 
1525  // Result is a load from the stack slot.
1526  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1527 }
1528 
1529 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1530 /// containing the sign bit if the target has no integer value capable of
1531 /// holding all bits of the floating-point value.
1532 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1533  const SDLoc &DL,
1534  SDValue Value) const {
1535  EVT FloatVT = Value.getValueType();
1536  unsigned NumBits = FloatVT.getScalarSizeInBits();
1537  State.FloatVT = FloatVT;
1538  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1539  // Convert to an integer of the same size.
1540  if (TLI.isTypeLegal(IVT)) {
1541  State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1542  State.SignMask = APInt::getSignMask(NumBits);
1543  State.SignBit = NumBits - 1;
1544  return;
1545  }
1546 
1547  auto &DataLayout = DAG.getDataLayout();
1548  // Store the float to memory, then load the sign part out as an integer.
1549  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1550  // First create a temporary that is aligned for both the load and store.
1551  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1552  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1553  // Then store the float to it.
1554  State.FloatPtr = StackPtr;
1555  MachineFunction &MF = DAG.getMachineFunction();
1556  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1557  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1558  State.FloatPointerInfo);
1559 
1560  SDValue IntPtr;
1561  if (DataLayout.isBigEndian()) {
1562  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1563  // Load out a legal integer with the same sign bit as the float.
1564  IntPtr = StackPtr;
1565  State.IntPointerInfo = State.FloatPointerInfo;
1566  } else {
1567  // Advance the pointer so that the loaded byte will contain the sign bit.
1568  unsigned ByteOffset = (NumBits / 8) - 1;
1569  IntPtr =
1570  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL);
1571  State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1572  ByteOffset);
1573  }
1574 
1575  State.IntPtr = IntPtr;
1576  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1577  State.IntPointerInfo, MVT::i8);
1578  State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1579  State.SignBit = 7;
1580 }
1581 
1582 /// Replace the integer value produced by getSignAsIntValue() with a new value
1583 /// and cast the result back to a floating-point type.
1584 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1585  const SDLoc &DL,
1586  SDValue NewIntValue) const {
1587  if (!State.Chain)
1588  return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1589 
1590  // Override the part containing the sign bit in the value stored on the stack.
1591  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1592  State.IntPointerInfo, MVT::i8);
1593  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1594  State.FloatPointerInfo);
1595 }
1596 
1597 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1598  SDLoc DL(Node);
1599  SDValue Mag = Node->getOperand(0);
1600  SDValue Sign = Node->getOperand(1);
1601 
1602  // Get sign bit into an integer value.
1603  FloatSignAsInt SignAsInt;
1604  getSignAsIntValue(SignAsInt, DL, Sign);
1605 
1606  EVT IntVT = SignAsInt.IntValue.getValueType();
1607  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1608  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1609  SignMask);
1610 
1611  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1612  EVT FloatVT = Mag.getValueType();
1613  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1614  TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1615  SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1616  SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1617  SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1618  DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1619  return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1620  }
1621 
1622  // Transform Mag value to integer, and clear the sign bit.
1623  FloatSignAsInt MagAsInt;
1624  getSignAsIntValue(MagAsInt, DL, Mag);
1625  EVT MagVT = MagAsInt.IntValue.getValueType();
1626  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1627  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1628  ClearSignMask);
1629 
1630  // Get the signbit at the right position for MagAsInt.
1631  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1632  EVT ShiftVT = IntVT;
1633  if (SignBit.getScalarValueSizeInBits() <
1634  ClearedSign.getScalarValueSizeInBits()) {
1635  SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1636  ShiftVT = MagVT;
1637  }
1638  if (ShiftAmount > 0) {
1639  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1640  SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1641  } else if (ShiftAmount < 0) {
1642  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1643  SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1644  }
1645  if (SignBit.getScalarValueSizeInBits() >
1646  ClearedSign.getScalarValueSizeInBits()) {
1647  SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1648  }
1649 
1650  // Store the part with the modified sign and convert back to float.
1651  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1652  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1653 }
1654 
1655 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1656  // Get the sign bit as an integer.
1657  SDLoc DL(Node);
1658  FloatSignAsInt SignAsInt;
1659  getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1660  EVT IntVT = SignAsInt.IntValue.getValueType();
1661 
1662  // Flip the sign.
1663  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1664  SDValue SignFlip =
1665  DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1666 
1667  // Convert back to float.
1668  return modifySignAsInt(SignAsInt, DL, SignFlip);
1669 }
1670 
1671 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1672  SDLoc DL(Node);
1673  SDValue Value = Node->getOperand(0);
1674 
1675  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1676  EVT FloatVT = Value.getValueType();
1677  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1678  SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1679  return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1680  }
1681 
1682  // Transform value to integer, clear the sign bit and transform back.
1683  FloatSignAsInt ValueAsInt;
1684  getSignAsIntValue(ValueAsInt, DL, Value);
1685  EVT IntVT = ValueAsInt.IntValue.getValueType();
1686  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1687  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1688  ClearSignMask);
1689  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1690 }
1691 
1692 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1694  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1695  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1696  " not tell us which reg is the stack pointer!");
1697  SDLoc dl(Node);
1698  EVT VT = Node->getValueType(0);
1699  SDValue Tmp1 = SDValue(Node, 0);
1700  SDValue Tmp2 = SDValue(Node, 1);
1701  SDValue Tmp3 = Node->getOperand(2);
1702  SDValue Chain = Tmp1.getOperand(0);
1703 
1704  // Chain the dynamic stack allocation so that it doesn't modify the stack
1705  // pointer when other instructions are using the stack.
1706  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1707 
1708  SDValue Size = Tmp2.getOperand(1);
1709  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1710  Chain = SP.getValue(1);
1711  Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1712  const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1713  unsigned Opc =
1715  ISD::ADD : ISD::SUB;
1716 
1717  Align StackAlign = TFL->getStackAlign();
1718  Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1719  if (Alignment > StackAlign)
1720  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1721  DAG.getConstant(-Alignment.value(), dl, VT));
1722  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1723 
1724  Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1725 
1726  Results.push_back(Tmp1);
1727  Results.push_back(Tmp2);
1728 }
1729 
1730 /// Emit a store/load combination to the stack. This stores
1731 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1732 /// a load from the stack slot to DestVT, extending it if needed.
1733 /// The resultant code need not be legal.
1734 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1735  EVT DestVT, const SDLoc &dl) {
1736  return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1737 }
1738 
1739 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1740  EVT DestVT, const SDLoc &dl,
1741  SDValue Chain) {
1742  EVT SrcVT = SrcOp.getValueType();
1743  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1744  Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1745 
1746  // Don't convert with stack if the load/store is expensive.
1747  if ((SrcVT.bitsGT(SlotVT) &&
1748  !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1749  (SlotVT.bitsLT(DestVT) &&
1750  !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1751  return SDValue();
1752 
1753  // Create the stack frame object.
1754  Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1755  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1756  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1757 
1758  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1759  int SPFI = StackPtrFI->getIndex();
1760  MachinePointerInfo PtrInfo =
1761  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1762 
1763  // Emit a store to the stack slot. Use a truncstore if the input value is
1764  // later than DestVT.
1765  SDValue Store;
1766 
1767  if (SrcVT.bitsGT(SlotVT))
1768  Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1769  SlotVT, SrcAlign);
1770  else {
1771  assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1772  Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1773  }
1774 
1775  // Result is a load from the stack slot.
1776  if (SlotVT.bitsEq(DestVT))
1777  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1778 
1779  assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1780  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1781  DestAlign);
1782 }
1783 
1784 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1785  SDLoc dl(Node);
1786  // Create a vector sized/aligned stack slot, store the value to element #0,
1787  // then load the whole vector back out.
1788  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1789 
1790  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1791  int SPFI = StackPtrFI->getIndex();
1792 
1793  SDValue Ch = DAG.getTruncStore(
1794  DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1795  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1796  Node->getValueType(0).getVectorElementType());
1797  return DAG.getLoad(
1798  Node->getValueType(0), dl, Ch, StackPtr,
1799  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1800 }
1801 
1802 static bool
1804  const TargetLowering &TLI, SDValue &Res) {
1805  unsigned NumElems = Node->getNumOperands();
1806  SDLoc dl(Node);
1807  EVT VT = Node->getValueType(0);
1808 
1809  // Try to group the scalars into pairs, shuffle the pairs together, then
1810  // shuffle the pairs of pairs together, etc. until the vector has
1811  // been built. This will work only if all of the necessary shuffle masks
1812  // are legal.
1813 
1814  // We do this in two phases; first to check the legality of the shuffles,
1815  // and next, assuming that all shuffles are legal, to create the new nodes.
1816  for (int Phase = 0; Phase < 2; ++Phase) {
1818  NewIntermedVals;
1819  for (unsigned i = 0; i < NumElems; ++i) {
1820  SDValue V = Node->getOperand(i);
1821  if (V.isUndef())
1822  continue;
1823 
1824  SDValue Vec;
1825  if (Phase)
1826  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1827  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1828  }
1829 
1830  while (IntermedVals.size() > 2) {
1831  NewIntermedVals.clear();
1832  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1833  // This vector and the next vector are shuffled together (simply to
1834  // append the one to the other).
1835  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1836 
1837  SmallVector<int, 16> FinalIndices;
1838  FinalIndices.reserve(IntermedVals[i].second.size() +
1839  IntermedVals[i+1].second.size());
1840 
1841  int k = 0;
1842  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1843  ++j, ++k) {
1844  ShuffleVec[k] = j;
1845  FinalIndices.push_back(IntermedVals[i].second[j]);
1846  }
1847  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1848  ++j, ++k) {
1849  ShuffleVec[k] = NumElems + j;
1850  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1851  }
1852 
1853  SDValue Shuffle;
1854  if (Phase)
1855  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1856  IntermedVals[i+1].first,
1857  ShuffleVec);
1858  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1859  return false;
1860  NewIntermedVals.push_back(
1861  std::make_pair(Shuffle, std::move(FinalIndices)));
1862  }
1863 
1864  // If we had an odd number of defined values, then append the last
1865  // element to the array of new vectors.
1866  if ((IntermedVals.size() & 1) != 0)
1867  NewIntermedVals.push_back(IntermedVals.back());
1868 
1869  IntermedVals.swap(NewIntermedVals);
1870  }
1871 
1872  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1873  "Invalid number of intermediate vectors");
1874  SDValue Vec1 = IntermedVals[0].first;
1875  SDValue Vec2;
1876  if (IntermedVals.size() > 1)
1877  Vec2 = IntermedVals[1].first;
1878  else if (Phase)
1879  Vec2 = DAG.getUNDEF(VT);
1880 
1881  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1882  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1883  ShuffleVec[IntermedVals[0].second[i]] = i;
1884  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1885  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1886 
1887  if (Phase)
1888  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1889  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1890  return false;
1891  }
1892 
1893  return true;
1894 }
1895 
1896 /// Expand a BUILD_VECTOR node on targets that don't
1897 /// support the operation, but do support the resultant vector type.
1898 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1899  unsigned NumElems = Node->getNumOperands();
1900  SDValue Value1, Value2;
1901  SDLoc dl(Node);
1902  EVT VT = Node->getValueType(0);
1903  EVT OpVT = Node->getOperand(0).getValueType();
1904  EVT EltVT = VT.getVectorElementType();
1905 
1906  // If the only non-undef value is the low element, turn this into a
1907  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1908  bool isOnlyLowElement = true;
1909  bool MoreThanTwoValues = false;
1910  bool isConstant = true;
1911  for (unsigned i = 0; i < NumElems; ++i) {
1912  SDValue V = Node->getOperand(i);
1913  if (V.isUndef())
1914  continue;
1915  if (i > 0)
1916  isOnlyLowElement = false;
1917  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1918  isConstant = false;
1919 
1920  if (!Value1.getNode()) {
1921  Value1 = V;
1922  } else if (!Value2.getNode()) {
1923  if (V != Value1)
1924  Value2 = V;
1925  } else if (V != Value1 && V != Value2) {
1926  MoreThanTwoValues = true;
1927  }
1928  }
1929 
1930  if (!Value1.getNode())
1931  return DAG.getUNDEF(VT);
1932 
1933  if (isOnlyLowElement)
1934  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1935 
1936  // If all elements are constants, create a load from the constant pool.
1937  if (isConstant) {
1939  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1940  if (ConstantFPSDNode *V =
1941  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1942  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1943  } else if (ConstantSDNode *V =
1944  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1945  if (OpVT==EltVT)
1946  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1947  else {
1948  // If OpVT and EltVT don't match, EltVT is not legal and the
1949  // element values have been promoted/truncated earlier. Undo this;
1950  // we don't want a v16i8 to become a v16i32 for example.
1951  const ConstantInt *CI = V->getConstantIntValue();
1952  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1953  CI->getZExtValue()));
1954  }
1955  } else {
1956  assert(Node->getOperand(i).isUndef());
1957  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1958  CV.push_back(UndefValue::get(OpNTy));
1959  }
1960  }
1962  SDValue CPIdx =
1963  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1964  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1965  return DAG.getLoad(
1966  VT, dl, DAG.getEntryNode(), CPIdx,
1967  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1968  Alignment);
1969  }
1970 
1971  SmallSet<SDValue, 16> DefinedValues;
1972  for (unsigned i = 0; i < NumElems; ++i) {
1973  if (Node->getOperand(i).isUndef())
1974  continue;
1975  DefinedValues.insert(Node->getOperand(i));
1976  }
1977 
1978  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1979  if (!MoreThanTwoValues) {
1980  SmallVector<int, 8> ShuffleVec(NumElems, -1);
1981  for (unsigned i = 0; i < NumElems; ++i) {
1982  SDValue V = Node->getOperand(i);
1983  if (V.isUndef())
1984  continue;
1985  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1986  }
1987  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1988  // Get the splatted value into the low element of a vector register.
1989  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1990  SDValue Vec2;
1991  if (Value2.getNode())
1992  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1993  else
1994  Vec2 = DAG.getUNDEF(VT);
1995 
1996  // Return shuffle(LowValVec, undef, <0,0,0,0>)
1997  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1998  }
1999  } else {
2000  SDValue Res;
2001  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2002  return Res;
2003  }
2004  }
2005 
2006  // Otherwise, we can't handle this case efficiently.
2007  return ExpandVectorBuildThroughStack(Node);
2008 }
2009 
2010 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2011  SDLoc DL(Node);
2012  EVT VT = Node->getValueType(0);
2013  SDValue SplatVal = Node->getOperand(0);
2014 
2015  return DAG.getSplatBuildVector(VT, DL, SplatVal);
2016 }
2017 
2018 // Expand a node into a call to a libcall. If the result value
2019 // does not fit into a register, return the lo part and set the hi part to the
2020 // by-reg argument. If it does fit into a single register, return the result
2021 // and leave the Hi part unset.
2022 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2023  bool isSigned) {
2025  TargetLowering::ArgListEntry Entry;
2026  for (const SDValue &Op : Node->op_values()) {
2027  EVT ArgVT = Op.getValueType();
2028  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2029  Entry.Node = Op;
2030  Entry.Ty = ArgTy;
2031  Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2032  Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2033  Args.push_back(Entry);
2034  }
2035  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2036  TLI.getPointerTy(DAG.getDataLayout()));
2037 
2038  EVT RetVT = Node->getValueType(0);
2039  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2040 
2041  // By default, the input chain to this libcall is the entry node of the
2042  // function. If the libcall is going to be emitted as a tail call then
2043  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2044  // node which is being folded has a non-entry input chain.
2045  SDValue InChain = DAG.getEntryNode();
2046 
2047  // isTailCall may be true since the callee does not reference caller stack
2048  // frame. Check if it's in the right position and that the return types match.
2049  SDValue TCChain = InChain;
2050  const Function &F = DAG.getMachineFunction().getFunction();
2051  bool isTailCall =
2052  TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2053  (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2054  if (isTailCall)
2055  InChain = TCChain;
2056 
2058  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2059  CLI.setDebugLoc(SDLoc(Node))
2060  .setChain(InChain)
2061  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2062  std::move(Args))
2063  .setTailCall(isTailCall)
2064  .setSExtResult(signExtend)
2065  .setZExtResult(!signExtend)
2066  .setIsPostTypeLegalization(true);
2067 
2068  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2069 
2070  if (!CallInfo.second.getNode()) {
2071  LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2072  // It's a tailcall, return the chain (which is the DAG root).
2073  return DAG.getRoot();
2074  }
2075 
2076  LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2077  return CallInfo.first;
2078 }
2079 
2080 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2081  RTLIB::Libcall LC,
2083  if (LC == RTLIB::UNKNOWN_LIBCALL)
2084  llvm_unreachable("Can't create an unknown libcall!");
2085 
2086  if (Node->isStrictFPOpcode()) {
2087  EVT RetVT = Node->getValueType(0);
2088  SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2090  // FIXME: This doesn't support tail calls.
2091  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2092  Ops, CallOptions,
2093  SDLoc(Node),
2094  Node->getOperand(0));
2095  Results.push_back(Tmp.first);
2096  Results.push_back(Tmp.second);
2097  } else {
2098  SDValue Tmp = ExpandLibCall(LC, Node, false);
2099  Results.push_back(Tmp);
2100  }
2101 }
2102 
2103 /// Expand the node to a libcall based on the result type.
2104 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2105  RTLIB::Libcall Call_F32,
2106  RTLIB::Libcall Call_F64,
2107  RTLIB::Libcall Call_F80,
2108  RTLIB::Libcall Call_F128,
2109  RTLIB::Libcall Call_PPCF128,
2111  RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2112  Call_F32, Call_F64, Call_F80,
2113  Call_F128, Call_PPCF128);
2114  ExpandFPLibCall(Node, LC, Results);
2115 }
2116 
2117 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2118  RTLIB::Libcall Call_I8,
2119  RTLIB::Libcall Call_I16,
2120  RTLIB::Libcall Call_I32,
2121  RTLIB::Libcall Call_I64,
2122  RTLIB::Libcall Call_I128) {
2123  RTLIB::Libcall LC;
2124  switch (Node->getSimpleValueType(0).SimpleTy) {
2125  default: llvm_unreachable("Unexpected request for libcall!");
2126  case MVT::i8: LC = Call_I8; break;
2127  case MVT::i16: LC = Call_I16; break;
2128  case MVT::i32: LC = Call_I32; break;
2129  case MVT::i64: LC = Call_I64; break;
2130  case MVT::i128: LC = Call_I128; break;
2131  }
2132  return ExpandLibCall(LC, Node, isSigned);
2133 }
2134 
2135 /// Expand the node to a libcall based on first argument type (for instance
2136 /// lround and its variant).
2137 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2138  RTLIB::Libcall Call_F32,
2139  RTLIB::Libcall Call_F64,
2140  RTLIB::Libcall Call_F80,
2141  RTLIB::Libcall Call_F128,
2142  RTLIB::Libcall Call_PPCF128,
2144  EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2146  Call_F32, Call_F64, Call_F80,
2147  Call_F128, Call_PPCF128);
2148  ExpandFPLibCall(Node, LC, Results);
2149 }
2150 
2151 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2152 void
2153 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2155  unsigned Opcode = Node->getOpcode();
2156  bool isSigned = Opcode == ISD::SDIVREM;
2157 
2158  RTLIB::Libcall LC;
2159  switch (Node->getSimpleValueType(0).SimpleTy) {
2160  default: llvm_unreachable("Unexpected request for libcall!");
2161  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2162  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2163  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2164  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2165  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2166  }
2167 
2168  // The input chain to this libcall is the entry node of the function.
2169  // Legalizing the call will automatically add the previous call to the
2170  // dependence.
2171  SDValue InChain = DAG.getEntryNode();
2172 
2173  EVT RetVT = Node->getValueType(0);
2174  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2175 
2177  TargetLowering::ArgListEntry Entry;
2178  for (const SDValue &Op : Node->op_values()) {
2179  EVT ArgVT = Op.getValueType();
2180  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2181  Entry.Node = Op;
2182  Entry.Ty = ArgTy;
2183  Entry.IsSExt = isSigned;
2184  Entry.IsZExt = !isSigned;
2185  Args.push_back(Entry);
2186  }
2187 
2188  // Also pass the return address of the remainder.
2189  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2190  Entry.Node = FIPtr;
2191  Entry.Ty = RetTy->getPointerTo();
2192  Entry.IsSExt = isSigned;
2193  Entry.IsZExt = !isSigned;
2194  Args.push_back(Entry);
2195 
2196  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2197  TLI.getPointerTy(DAG.getDataLayout()));
2198 
2199  SDLoc dl(Node);
2201  CLI.setDebugLoc(dl)
2202  .setChain(InChain)
2203  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2204  std::move(Args))
2205  .setSExtResult(isSigned)
2206  .setZExtResult(!isSigned);
2207 
2208  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2209 
2210  // Remainder is loaded back from the stack frame.
2211  SDValue Rem =
2212  DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2213  Results.push_back(CallInfo.first);
2214  Results.push_back(Rem);
2215 }
2216 
2217 /// Return true if sincos libcall is available.
2218 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2219  RTLIB::Libcall LC;
2220  switch (Node->getSimpleValueType(0).SimpleTy) {
2221  default: llvm_unreachable("Unexpected request for libcall!");
2222  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2223  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2224  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2225  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2226  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2227  }
2228  return TLI.getLibcallName(LC) != nullptr;
2229 }
2230 
2231 /// Only issue sincos libcall if both sin and cos are needed.
2232 static bool useSinCos(SDNode *Node) {
2233  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2234  ? ISD::FCOS : ISD::FSIN;
2235 
2236  SDValue Op0 = Node->getOperand(0);
2237  for (const SDNode *User : Op0.getNode()->uses()) {
2238  if (User == Node)
2239  continue;
2240  // The other user might have been turned into sincos already.
2241  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2242  return true;
2243  }
2244  return false;
2245 }
2246 
2247 /// Issue libcalls to sincos to compute sin / cos pairs.
2248 void
2249 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2251  RTLIB::Libcall LC;
2252  switch (Node->getSimpleValueType(0).SimpleTy) {
2253  default: llvm_unreachable("Unexpected request for libcall!");
2254  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2255  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2256  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2257  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2258  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2259  }
2260 
2261  // The input chain to this libcall is the entry node of the function.
2262  // Legalizing the call will automatically add the previous call to the
2263  // dependence.
2264  SDValue InChain = DAG.getEntryNode();
2265 
2266  EVT RetVT = Node->getValueType(0);
2267  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2268 
2270  TargetLowering::ArgListEntry Entry;
2271 
2272  // Pass the argument.
2273  Entry.Node = Node->getOperand(0);
2274  Entry.Ty = RetTy;
2275  Entry.IsSExt = false;
2276  Entry.IsZExt = false;
2277  Args.push_back(Entry);
2278 
2279  // Pass the return address of sin.
2280  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2281  Entry.Node = SinPtr;
2282  Entry.Ty = RetTy->getPointerTo();
2283  Entry.IsSExt = false;
2284  Entry.IsZExt = false;
2285  Args.push_back(Entry);
2286 
2287  // Also pass the return address of the cos.
2288  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2289  Entry.Node = CosPtr;
2290  Entry.Ty = RetTy->getPointerTo();
2291  Entry.IsSExt = false;
2292  Entry.IsZExt = false;
2293  Args.push_back(Entry);
2294 
2295  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2296  TLI.getPointerTy(DAG.getDataLayout()));
2297 
2298  SDLoc dl(Node);
2300  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2301  TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2302  std::move(Args));
2303 
2304  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2305 
2306  Results.push_back(
2307  DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2308  Results.push_back(
2309  DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2310 }
2311 
2312 /// This function is responsible for legalizing a
2313 /// INT_TO_FP operation of the specified operand when the target requests that
2314 /// we expand it. At this point, we know that the result and operand types are
2315 /// legal for the target.
2316 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2317  SDValue &Chain) {
2318  bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2319  Node->getOpcode() == ISD::SINT_TO_FP);
2320  EVT DestVT = Node->getValueType(0);
2321  SDLoc dl(Node);
2322  unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2323  SDValue Op0 = Node->getOperand(OpNo);
2324  EVT SrcVT = Op0.getValueType();
2325 
2326  // TODO: Should any fast-math-flags be set for the created nodes?
2327  LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2328  if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2329  (DestVT.bitsLE(MVT::f64) ||
2330  TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2331  : ISD::FP_EXTEND,
2332  DestVT))) {
2333  LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2334  "expansion\n");
2335 
2336  // Get the stack frame index of a 8 byte buffer.
2337  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2338 
2339  SDValue Lo = Op0;
2340  // if signed map to unsigned space
2341  if (isSigned) {
2342  // Invert sign bit (signed to unsigned mapping).
2343  Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2344  DAG.getConstant(0x80000000u, dl, MVT::i32));
2345  }
2346  // Initial hi portion of constructed double.
2347  SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2348 
2349  // If this a big endian target, swap the lo and high data.
2350  if (DAG.getDataLayout().isBigEndian())
2351  std::swap(Lo, Hi);
2352 
2353  SDValue MemChain = DAG.getEntryNode();
2354 
2355  // Store the lo of the constructed double.
2356  SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2357  MachinePointerInfo());
2358  // Store the hi of the constructed double.
2359  SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl);
2360  SDValue Store2 =
2361  DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2362  MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2363 
2364  // load the constructed double
2365  SDValue Load =
2366  DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2367  // FP constant to bias correct the final result
2368  SDValue Bias = DAG.getConstantFP(isSigned ?
2369  BitsToDouble(0x4330000080000000ULL) :
2370  BitsToDouble(0x4330000000000000ULL),
2371  dl, MVT::f64);
2372  // Subtract the bias and get the final result.
2373  SDValue Sub;
2374  SDValue Result;
2375  if (Node->isStrictFPOpcode()) {
2376  Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2377  {Node->getOperand(0), Load, Bias});
2378  Chain = Sub.getValue(1);
2379  if (DestVT != Sub.getValueType()) {
2380  std::pair<SDValue, SDValue> ResultPair;
2381  ResultPair =
2382  DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2383  Result = ResultPair.first;
2384  Chain = ResultPair.second;
2385  }
2386  else
2387  Result = Sub;
2388  } else {
2389  Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2390  Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2391  }
2392  return Result;
2393  }
2394 
2395  if (isSigned)
2396  return SDValue();
2397 
2398  // TODO: Generalize this for use with other types.
2399  if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2400  (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2401  LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2402  // For unsigned conversions, convert them to signed conversions using the
2403  // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2404  // should be valid for i32->f32 as well.
2405 
2406  // More generally this transform should be valid if there are 3 more bits
2407  // in the integer type than the significand. Rounding uses the first bit
2408  // after the width of the significand and the OR of all bits after that. So
2409  // we need to be able to OR the shifted out bit into one of the bits that
2410  // participate in the OR.
2411 
2412  // TODO: This really should be implemented using a branch rather than a
2413  // select. We happen to get lucky and machinesink does the right
2414  // thing most of the time. This would be a good candidate for a
2415  // pseudo-op, or, even better, for whole-function isel.
2416  EVT SetCCVT = getSetCCResultType(SrcVT);
2417 
2418  SDValue SignBitTest = DAG.getSetCC(
2419  dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2420 
2421  EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2422  SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2423  SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2424  SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2425  SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2426  SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2427 
2428  SDValue Slow, Fast;
2429  if (Node->isStrictFPOpcode()) {
2430  // In strict mode, we must avoid spurious exceptions, and therefore
2431  // must make sure to only emit a single STRICT_SINT_TO_FP.
2432  SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2433  Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2434  { Node->getOperand(0), InCvt });
2435  Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2436  { Fast.getValue(1), Fast, Fast });
2437  Chain = Slow.getValue(1);
2438  // The STRICT_SINT_TO_FP inherits the exception mode from the
2439  // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2440  // never raise any exception.
2442  Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2443  Fast->setFlags(Flags);
2444  Flags.setNoFPExcept(true);
2445  Slow->setFlags(Flags);
2446  } else {
2447  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2448  Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2449  Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2450  }
2451 
2452  return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2453  }
2454 
2455  // Don't expand it if there isn't cheap fadd.
2456  if (!TLI.isOperationLegalOrCustom(
2457  Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2458  return SDValue();
2459 
2460  // The following optimization is valid only if every value in SrcVT (when
2461  // treated as signed) is representable in DestVT. Check that the mantissa
2462  // size of DestVT is >= than the number of bits in SrcVT -1.
2463  assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2464  SrcVT.getSizeInBits() - 1 &&
2465  "Cannot perform lossless SINT_TO_FP!");
2466 
2467  SDValue Tmp1;
2468  if (Node->isStrictFPOpcode()) {
2469  Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2470  { Node->getOperand(0), Op0 });
2471  } else
2472  Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2473 
2474  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2475  DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2476  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2477  Four = DAG.getIntPtrConstant(4, dl);
2478  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2479  SignSet, Four, Zero);
2480 
2481  // If the sign bit of the integer is set, the large number will be treated
2482  // as a negative number. To counteract this, the dynamic code adds an
2483  // offset depending on the data type.
2484  uint64_t FF;
2485  switch (SrcVT.getSimpleVT().SimpleTy) {
2486  default:
2487  return SDValue();
2488  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2489  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2490  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2491  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2492  }
2493  if (DAG.getDataLayout().isLittleEndian())
2494  FF <<= 32;
2495  Constant *FudgeFactor = ConstantInt::get(
2496  Type::getInt64Ty(*DAG.getContext()), FF);
2497 
2498  SDValue CPIdx =
2499  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2500  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2501  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2502  Alignment = commonAlignment(Alignment, 4);
2503  SDValue FudgeInReg;
2504  if (DestVT == MVT::f32)
2505  FudgeInReg = DAG.getLoad(
2506  MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2507  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2508  Alignment);
2509  else {
2510  SDValue Load = DAG.getExtLoad(
2511  ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2512  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2513  Alignment);
2514  HandleSDNode Handle(Load);
2515  LegalizeOp(Load.getNode());
2516  FudgeInReg = Handle.getValue();
2517  }
2518 
2519  if (Node->isStrictFPOpcode()) {
2520  SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2521  { Tmp1.getValue(1), Tmp1, FudgeInReg });
2522  Chain = Result.getValue(1);
2523  return Result;
2524  }
2525 
2526  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2527 }
2528 
2529 /// This function is responsible for legalizing a
2530 /// *INT_TO_FP operation of the specified operand when the target requests that
2531 /// we promote it. At this point, we know that the result and operand types are
2532 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2533 /// operation that takes a larger input.
2534 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2535  SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2536  bool IsStrict = N->isStrictFPOpcode();
2537  bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2538  N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2539  EVT DestVT = N->getValueType(0);
2540  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2541  unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2542  unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2543 
2544  // First step, figure out the appropriate *INT_TO_FP operation to use.
2545  EVT NewInTy = LegalOp.getValueType();
2546 
2547  unsigned OpToUse = 0;
2548 
2549  // Scan for the appropriate larger type to use.
2550  while (true) {
2551  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2552  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2553 
2554  // If the target supports SINT_TO_FP of this type, use it.
2555  if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2556  OpToUse = SIntOp;
2557  break;
2558  }
2559  if (IsSigned)
2560  continue;
2561 
2562  // If the target supports UINT_TO_FP of this type, use it.
2563  if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2564  OpToUse = UIntOp;
2565  break;
2566  }
2567 
2568  // Otherwise, try a larger type.
2569  }
2570 
2571  // Okay, we found the operation and type to use. Zero extend our input to the
2572  // desired type then run the operation on it.
2573  if (IsStrict) {
2574  SDValue Res =
2575  DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2576  {N->getOperand(0),
2577  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2578  dl, NewInTy, LegalOp)});
2579  Results.push_back(Res);
2580  Results.push_back(Res.getValue(1));
2581  return;
2582  }
2583 
2584  Results.push_back(
2585  DAG.getNode(OpToUse, dl, DestVT,
2586  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2587  dl, NewInTy, LegalOp)));
2588 }
2589 
2590 /// This function is responsible for legalizing a
2591 /// FP_TO_*INT operation of the specified operand when the target requests that
2592 /// we promote it. At this point, we know that the result and operand types are
2593 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2594 /// operation that returns a larger result.
2595 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2597  bool IsStrict = N->isStrictFPOpcode();
2598  bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2599  N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2600  EVT DestVT = N->getValueType(0);
2601  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2602  // First step, figure out the appropriate FP_TO*INT operation to use.
2603  EVT NewOutTy = DestVT;
2604 
2605  unsigned OpToUse = 0;
2606 
2607  // Scan for the appropriate larger type to use.
2608  while (true) {
2609  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2610  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2611 
2612  // A larger signed type can hold all unsigned values of the requested type,
2613  // so using FP_TO_SINT is valid
2614  OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2615  if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2616  break;
2617 
2618  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2619  OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2620  if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2621  break;
2622 
2623  // Otherwise, try a larger type.
2624  }
2625 
2626  // Okay, we found the operation and type to use.
2628  if (IsStrict) {
2629  SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2630  Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2631  } else
2632  Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2633 
2634  // Truncate the result of the extended FP_TO_*INT operation to the desired
2635  // size.
2636  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2637  Results.push_back(Trunc);
2638  if (IsStrict)
2639  Results.push_back(Operation.getValue(1));
2640 }
2641 
2642 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2643 /// the result and operand types are legal and there must be a legal
2644 /// FP_TO_*INT_SAT operation for a larger result type.
2645 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2646  const SDLoc &dl) {
2647  unsigned Opcode = Node->getOpcode();
2648 
2649  // Scan for the appropriate larger type to use.
2650  EVT NewOutTy = Node->getValueType(0);
2651  while (true) {
2652  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2653  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2654 
2655  if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2656  break;
2657  }
2658 
2659  // Saturation width is determined by second operand, so we don't have to
2660  // perform any fixup and can directly truncate the result.
2661  SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2662  Node->getOperand(1));
2663  return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2664 }
2665 
2666 /// Open code the operations for PARITY of the specified operation.
2667 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2668  EVT VT = Op.getValueType();
2669  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2670  unsigned Sz = VT.getScalarSizeInBits();
2671 
2672  // If CTPOP is legal, use it. Otherwise use shifts and xor.
2673  SDValue Result;
2674  if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2675  Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2676  } else {
2677  Result = Op;
2678  for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2679  SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2680  DAG.getConstant(1ULL << (--i), dl, ShVT));
2681  Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2682  }
2683  }
2684 
2685  return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2686 }
2687 
2688 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2689  LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2691  SDLoc dl(Node);
2692  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2693  bool NeedInvert;
2694  switch (Node->getOpcode()) {
2695  case ISD::ABS:
2696  if ((Tmp1 = TLI.expandABS(Node, DAG)))
2697  Results.push_back(Tmp1);
2698  break;
2699  case ISD::CTPOP:
2700  if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
2701  Results.push_back(Tmp1);
2702  break;
2703  case ISD::CTLZ:
2704  case ISD::CTLZ_ZERO_UNDEF:
2705  if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
2706  Results.push_back(Tmp1);
2707  break;
2708  case ISD::CTTZ:
2709  case ISD::CTTZ_ZERO_UNDEF:
2710  if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
2711  Results.push_back(Tmp1);
2712  break;
2713  case ISD::BITREVERSE:
2714  if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
2715  Results.push_back(Tmp1);
2716  break;
2717  case ISD::BSWAP:
2718  if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
2719  Results.push_back(Tmp1);
2720  break;
2721  case ISD::PARITY:
2722  Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
2723  break;
2724  case ISD::FRAMEADDR:
2725  case ISD::RETURNADDR:
2727  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2728  break;
2729  case ISD::EH_DWARF_CFA: {
2730  SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2731  TLI.getPointerTy(DAG.getDataLayout()));
2732  SDValue Offset = DAG.getNode(ISD::ADD, dl,
2733  CfaArg.getValueType(),
2734  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2735  CfaArg.getValueType()),
2736  CfaArg);
2737  SDValue FA = DAG.getNode(
2738  ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2739  DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2740  Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2741  FA, Offset));
2742  break;
2743  }
2744  case ISD::FLT_ROUNDS_:
2745  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2746  Results.push_back(Node->getOperand(0));
2747  break;
2748  case ISD::EH_RETURN:
2749  case ISD::EH_LABEL:
2750  case ISD::PREFETCH:
2751  case ISD::VAEND:
2752  case ISD::EH_SJLJ_LONGJMP:
2753  // If the target didn't expand these, there's nothing to do, so just
2754  // preserve the chain and be done.
2755  Results.push_back(Node->getOperand(0));
2756  break;
2757  case ISD::READCYCLECOUNTER:
2758  // If the target didn't expand this, just return 'zero' and preserve the
2759  // chain.
2760  Results.append(Node->getNumValues() - 1,
2761  DAG.getConstant(0, dl, Node->getValueType(0)));
2762  Results.push_back(Node->getOperand(0));
2763  break;
2764  case ISD::EH_SJLJ_SETJMP:
2765  // If the target didn't expand this, just return 'zero' and preserve the
2766  // chain.
2767  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2768  Results.push_back(Node->getOperand(0));
2769  break;
2770  case ISD::ATOMIC_LOAD: {
2771  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2772  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2773  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2774  SDValue Swap = DAG.getAtomicCmpSwap(
2775  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2776  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2777  cast<AtomicSDNode>(Node)->getMemOperand());
2778  Results.push_back(Swap.getValue(0));
2779  Results.push_back(Swap.getValue(1));
2780  break;
2781  }
2782  case ISD::ATOMIC_STORE: {
2783  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2784  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2785  cast<AtomicSDNode>(Node)->getMemoryVT(),
2786  Node->getOperand(0),
2787  Node->getOperand(1), Node->getOperand(2),
2788  cast<AtomicSDNode>(Node)->getMemOperand());
2789  Results.push_back(Swap.getValue(1));
2790  break;
2791  }
2793  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2794  // splits out the success value as a comparison. Expanding the resulting
2795  // ATOMIC_CMP_SWAP will produce a libcall.
2796  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2797  SDValue Res = DAG.getAtomicCmpSwap(
2798  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2799  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2800  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2801 
2802  SDValue ExtRes = Res;
2803  SDValue LHS = Res;
2804  SDValue RHS = Node->getOperand(1);
2805 
2806  EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2807  EVT OuterType = Node->getValueType(0);
2808  switch (TLI.getExtendForAtomicOps()) {
2809  case ISD::SIGN_EXTEND:
2810  LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2811  DAG.getValueType(AtomicType));
2812  RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2813  Node->getOperand(2), DAG.getValueType(AtomicType));
2814  ExtRes = LHS;
2815  break;
2816  case ISD::ZERO_EXTEND:
2817  LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2818  DAG.getValueType(AtomicType));
2819  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2820  ExtRes = LHS;
2821  break;
2822  case ISD::ANY_EXTEND:
2823  LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2824  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2825  break;
2826  default:
2827  llvm_unreachable("Invalid atomic op extension");
2828  }
2829 
2830  SDValue Success =
2831  DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2832 
2833  Results.push_back(ExtRes.getValue(0));
2834  Results.push_back(Success);
2835  Results.push_back(Res.getValue(1));
2836  break;
2837  }
2839  ExpandDYNAMIC_STACKALLOC(Node, Results);
2840  break;
2841  case ISD::MERGE_VALUES:
2842  for (unsigned i = 0; i < Node->getNumValues(); i++)
2843  Results.push_back(Node->getOperand(i));
2844  break;
2845  case ISD::UNDEF: {
2846  EVT VT = Node->getValueType(0);
2847  if (VT.isInteger())
2848  Results.push_back(DAG.getConstant(0, dl, VT));
2849  else {
2850  assert(VT.isFloatingPoint() && "Unknown value type!");
2851  Results.push_back(DAG.getConstantFP(0, dl, VT));
2852  }
2853  break;
2854  }
2855  case ISD::STRICT_FP_ROUND:
2856  // When strict mode is enforced we can't do expansion because it
2857  // does not honor the "strict" properties. Only libcall is allowed.
2858  if (TLI.isStrictFPEnabled())
2859  break;
2860  // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
2861  // since this operation is more efficient than stack operation.
2862  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2863  Node->getValueType(0))
2865  break;
2866  // We fall back to use stack operation when the FP_ROUND operation
2867  // isn't available.
2868  if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
2869  Node->getValueType(0), dl,
2870  Node->getOperand(0)))) {
2871  ReplaceNode(Node, Tmp1.getNode());
2872  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2873  return true;
2874  }
2875  break;
2876  case ISD::FP_ROUND:
2877  case ISD::BITCAST:
2878  if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2879  Node->getValueType(0), dl)))
2880  Results.push_back(Tmp1);
2881  break;
2882  case ISD::STRICT_FP_EXTEND:
2883  // When strict mode is enforced we can't do expansion because it
2884  // does not honor the "strict" properties. Only libcall is allowed.
2885  if (TLI.isStrictFPEnabled())
2886  break;
2887  // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
2888  // since this operation is more efficient than stack operation.
2889  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2890  Node->getValueType(0))
2892  break;
2893  // We fall back to use stack operation when the FP_EXTEND operation
2894  // isn't available.
2895  if ((Tmp1 = EmitStackConvert(
2896  Node->getOperand(1), Node->getOperand(1).getValueType(),
2897  Node->getValueType(0), dl, Node->getOperand(0)))) {
2898  ReplaceNode(Node, Tmp1.getNode());
2899  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2900  return true;
2901  }
2902  break;
2903  case ISD::FP_EXTEND:
2904  if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
2905  Node->getOperand(0).getValueType(),
2906  Node->getValueType(0), dl)))
2907  Results.push_back(Tmp1);
2908  break;
2909  case ISD::BF16_TO_FP: {
2910  // Always expand bf16 to f32 casts, they lower to ext + shift.
2911  SDValue Op = DAG.getNode(ISD::BITCAST, dl, MVT::i16, Node->getOperand(0));
2912  Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op);
2913  Op = DAG.getNode(
2914  ISD::SHL, dl, MVT::i32, Op,
2915  DAG.getConstant(16, dl,
2916  TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
2917  Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
2918  // Add fp_extend in case the output is bigger than f32.
2919  if (Node->getValueType(0) != MVT::f32)
2920  Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
2921  Results.push_back(Op);
2922  break;
2923  }
2924  case ISD::SIGN_EXTEND_INREG: {
2925  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2926  EVT VT = Node->getValueType(0);
2927 
2928  // An in-register sign-extend of a boolean is a negation:
2929  // 'true' (1) sign-extended is -1.
2930  // 'false' (0) sign-extended is 0.
2931  // However, we must mask the high bits of the source operand because the
2932  // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2933 
2934  // TODO: Do this for vectors too?
2935  if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
2936  SDValue One = DAG.getConstant(1, dl, VT);
2937  SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2938  SDValue Zero = DAG.getConstant(0, dl, VT);
2939  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2940  Results.push_back(Neg);
2941  break;
2942  }
2943 
2944  // NOTE: we could fall back on load/store here too for targets without
2945  // SRA. However, it is doubtful that any exist.
2946  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2947  unsigned BitsDiff = VT.getScalarSizeInBits() -
2948  ExtraVT.getScalarSizeInBits();
2949  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2950  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2951  Node->getOperand(0), ShiftCst);
2952  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2953  Results.push_back(Tmp1);
2954  break;
2955  }
2956  case ISD::UINT_TO_FP:
2958  if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
2959  Results.push_back(Tmp1);
2960  if (Node->isStrictFPOpcode())
2961  Results.push_back(Tmp2);
2962  break;
2963  }
2964  [[fallthrough]];
2965  case ISD::SINT_TO_FP:
2967  if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
2968  Results.push_back(Tmp1);
2969  if (Node->isStrictFPOpcode())
2970  Results.push_back(Tmp2);
2971  }
2972  break;
2973  case ISD::FP_TO_SINT:
2974  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2975  Results.push_back(Tmp1);
2976  break;
2978  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2979  ReplaceNode(Node, Tmp1.getNode());
2980  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
2981  return true;
2982  }
2983  break;
2984  case ISD::FP_TO_UINT:
2985  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2986  Results.push_back(Tmp1);
2987  break;
2989  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2990  // Relink the chain.
2991  DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
2992  // Replace the new UINT result.
2993  ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
2994  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
2995  return true;
2996  }
2997  break;
2998  case ISD::FP_TO_SINT_SAT:
2999  case ISD::FP_TO_UINT_SAT:
3000  Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3001  break;
3002  case ISD::VAARG:
3003  Results.push_back(DAG.expandVAArg(Node));
3004  Results.push_back(Results[0].getValue(1));
3005  break;
3006  case ISD::VACOPY:
3007  Results.push_back(DAG.expandVACopy(Node));
3008  break;
3010  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3011  // This must be an access of the only element. Return it.
3012  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3013  Node->getOperand(0));
3014  else
3015  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3016  Results.push_back(Tmp1);
3017  break;
3019  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3020  break;
3021  case ISD::INSERT_SUBVECTOR:
3022  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3023  break;
3024  case ISD::CONCAT_VECTORS:
3025  Results.push_back(ExpandVectorBuildThroughStack(Node));
3026  break;
3027  case ISD::SCALAR_TO_VECTOR:
3028  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3029  break;
3031  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3032  Node->getOperand(1),
3033  Node->getOperand(2), dl));
3034  break;
3035  case ISD::VECTOR_SHUFFLE: {
3036  SmallVector<int, 32> NewMask;
3037  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3038 
3039  EVT VT = Node->getValueType(0);
3040  EVT EltVT = VT.getVectorElementType();
3041  SDValue Op0 = Node->getOperand(0);
3042  SDValue Op1 = Node->getOperand(1);
3043  if (!TLI.isTypeLegal(EltVT)) {
3044  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3045 
3046  // BUILD_VECTOR operands are allowed to be wider than the element type.
3047  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3048  // it.
3049  if (NewEltVT.bitsLT(EltVT)) {
3050  // Convert shuffle node.
3051  // If original node was v4i64 and the new EltVT is i32,
3052  // cast operands to v8i32 and re-build the mask.
3053 
3054  // Calculate new VT, the size of the new VT should be equal to original.
3055  EVT NewVT =
3056  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3057  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3058  assert(NewVT.bitsEq(VT));
3059 
3060  // cast operands to new VT
3061  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3062  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3063 
3064  // Convert the shuffle mask
3065  unsigned int factor =
3067 
3068  // EltVT gets smaller
3069  assert(factor > 0);
3070 
3071  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3072  if (Mask[i] < 0) {
3073  for (unsigned fi = 0; fi < factor; ++fi)
3074  NewMask.push_back(Mask[i]);
3075  }
3076  else {
3077  for (unsigned fi = 0; fi < factor; ++fi)
3078  NewMask.push_back(Mask[i]*factor+fi);
3079  }
3080  }
3081  Mask = NewMask;
3082  VT = NewVT;
3083  }
3084  EltVT = NewEltVT;
3085  }
3086  unsigned NumElems = VT.getVectorNumElements();
3088  for (unsigned i = 0; i != NumElems; ++i) {
3089  if (Mask[i] < 0) {
3090  Ops.push_back(DAG.getUNDEF(EltVT));
3091  continue;
3092  }
3093  unsigned Idx = Mask[i];
3094  if (Idx < NumElems)
3095  Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3096  DAG.getVectorIdxConstant(Idx, dl)));
3097  else
3098  Ops.push_back(
3099  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3100  DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3101  }
3102 
3103  Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3104  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3105  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3106  Results.push_back(Tmp1);
3107  break;
3108  }
3109  case ISD::VECTOR_SPLICE: {
3110  Results.push_back(TLI.expandVectorSplice(Node, DAG));
3111  break;
3112  }
3113  case ISD::EXTRACT_ELEMENT: {
3114  EVT OpTy = Node->getOperand(0).getValueType();
3115  if (Node->getConstantOperandVal(1)) {
3116  // 1 -> Hi
3117  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3118  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3119  TLI.getShiftAmountTy(
3120  Node->getOperand(0).getValueType(),
3121  DAG.getDataLayout())));
3122  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3123  } else {
3124  // 0 -> Lo
3125  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3126  Node->getOperand(0));
3127  }
3128  Results.push_back(Tmp1);
3129  break;
3130  }
3131  case ISD::STACKSAVE:
3132  // Expand to CopyFromReg if the target set
3133  // StackPointerRegisterToSaveRestore.
3134  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3135  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3136  Node->getValueType(0)));
3137  Results.push_back(Results[0].getValue(1));
3138  } else {
3139  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3140  Results.push_back(Node->getOperand(0));
3141  }
3142  break;
3143  case ISD::STACKRESTORE:
3144  // Expand to CopyToReg if the target set
3145  // StackPointerRegisterToSaveRestore.
3146  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3147  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3148  Node->getOperand(1)));
3149  } else {
3150  Results.push_back(Node->getOperand(0));
3151  }
3152  break;
3154  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3155  Results.push_back(Results[0].getValue(0));
3156  break;
3157  case ISD::FCOPYSIGN:
3158  Results.push_back(ExpandFCOPYSIGN(Node));
3159  break;
3160  case ISD::FNEG:
3161  Results.push_back(ExpandFNEG(Node));
3162  break;
3163  case ISD::FABS:
3164  Results.push_back(ExpandFABS(Node));
3165  break;
3166  case ISD::IS_FPCLASS: {
3167  auto CNode = cast<ConstantSDNode>(Node->getOperand(1));
3168  auto Test = static_cast<FPClassTest>(CNode->getZExtValue());
3169  if (SDValue Expanded =
3170  TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3171  Test, Node->getFlags(), SDLoc(Node), DAG))
3172  Results.push_back(Expanded);
3173  break;
3174  }
3175  case ISD::SMIN:
3176  case ISD::SMAX:
3177  case ISD::UMIN:
3178  case ISD::UMAX: {
3179  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3180  ISD::CondCode Pred;
3181  switch (Node->getOpcode()) {
3182  default: llvm_unreachable("How did we get here?");
3183  case ISD::SMAX: Pred = ISD::SETGT; break;
3184  case ISD::SMIN: Pred = ISD::SETLT; break;
3185  case ISD::UMAX: Pred = ISD::SETUGT; break;
3186  case ISD::UMIN: Pred = ISD::SETULT; break;
3187  }
3188  Tmp1 = Node->getOperand(0);
3189  Tmp2 = Node->getOperand(1);
3190  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3191  Results.push_back(Tmp1);
3192  break;
3193  }
3194  case ISD::FMINNUM:
3195  case ISD::FMAXNUM: {
3196  if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3197  Results.push_back(Expanded);
3198  break;
3199  }
3200  case ISD::FSIN:
3201  case ISD::FCOS: {
3202  EVT VT = Node->getValueType(0);
3203  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3204  // fcos which share the same operand and both are used.
3205  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3206  isSinCosLibcallAvailable(Node, TLI))
3207  && useSinCos(Node)) {
3208  SDVTList VTs = DAG.getVTList(VT, VT);
3209  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3210  if (Node->getOpcode() == ISD::FCOS)
3211  Tmp1 = Tmp1.getValue(1);
3212  Results.push_back(Tmp1);
3213  }
3214  break;
3215  }
3216  case ISD::FMAD:
3217  llvm_unreachable("Illegal fmad should never be formed");
3218 
3219  case ISD::FP16_TO_FP:
3220  if (Node->getValueType(0) != MVT::f32) {
3221  // We can extend to types bigger than f32 in two steps without changing
3222  // the result. Since "f16 -> f32" is much more commonly available, give
3223  // CodeGen the option of emitting that before resorting to a libcall.
3224  SDValue Res =
3225  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3226  Results.push_back(
3227  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3228  }
3229  break;
3231  if (Node->getValueType(0) != MVT::f32) {
3232  // We can extend to types bigger than f32 in two steps without changing
3233  // the result. Since "f16 -> f32" is much more commonly available, give
3234  // CodeGen the option of emitting that before resorting to a libcall.
3235  SDValue Res =
3237  {Node->getOperand(0), Node->getOperand(1)});
3238  Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3239  {Node->getValueType(0), MVT::Other},
3240  {Res.getValue(1), Res});
3241  Results.push_back(Res);
3242  Results.push_back(Res.getValue(1));
3243  }
3244  break;
3245  case ISD::FP_TO_FP16:
3246  LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3247  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3248  SDValue Op = Node->getOperand(0);
3249  MVT SVT = Op.getSimpleValueType();
3250  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3251  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3252  // Under fastmath, we can expand this node into a fround followed by
3253  // a float-half conversion.
3254  SDValue FloatVal =
3255  DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3256  DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3257  Results.push_back(
3258  DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3259  }
3260  }
3261  break;
3262  case ISD::ConstantFP: {
3263  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3264  // Check to see if this FP immediate is already legal.
3265  // If this is a legal constant, turn it into a TargetConstantFP node.
3266  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3267  DAG.shouldOptForSize()))
3268  Results.push_back(ExpandConstantFP(CFP, true));
3269  break;
3270  }
3271  case ISD::Constant: {
3272  ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3273  Results.push_back(ExpandConstant(CP));
3274  break;
3275  }
3276  case ISD::FSUB: {
3277  EVT VT = Node->getValueType(0);
3278  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3279  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3280  const SDNodeFlags Flags = Node->getFlags();
3281  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3282  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3283  Results.push_back(Tmp1);
3284  }
3285  break;
3286  }
3287  case ISD::SUB: {
3288  EVT VT = Node->getValueType(0);
3289  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3290  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3291  "Don't know how to expand this subtraction!");
3292  Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3293  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3294  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3295  break;
3296  }
3297  case ISD::UREM:
3298  case ISD::SREM:
3299  if (TLI.expandREM(Node, Tmp1, DAG))
3300  Results.push_back(Tmp1);
3301  break;
3302  case ISD::UDIV:
3303  case ISD::SDIV: {
3304  bool isSigned = Node->getOpcode() == ISD::SDIV;
3305  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3306  EVT VT = Node->getValueType(0);
3307  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3308  SDVTList VTs = DAG.getVTList(VT, VT);
3309  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3310  Node->getOperand(1));
3311  Results.push_back(Tmp1);
3312  }
3313  break;
3314  }
3315  case ISD::MULHU:
3316  case ISD::MULHS: {
3317  unsigned ExpandOpcode =
3318  Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3319  EVT VT = Node->getValueType(0);
3320  SDVTList VTs = DAG.getVTList(VT, VT);
3321 
3322  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3323  Node->getOperand(1));
3324  Results.push_back(Tmp1.getValue(1));
3325  break;
3326  }
3327  case ISD::UMUL_LOHI:
3328  case ISD::SMUL_LOHI: {
3329  SDValue LHS = Node->getOperand(0);
3330  SDValue RHS = Node->getOperand(1);
3331  MVT VT = LHS.getSimpleValueType();
3332  unsigned MULHOpcode =
3333  Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3334 
3335  if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3336  Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3337  Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3338  break;
3339  }
3340 
3341  SmallVector<SDValue, 4> Halves;
3342  EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3343  assert(TLI.isTypeLegal(HalfType));
3344  if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3345  HalfType, DAG,
3347  for (unsigned i = 0; i < 2; ++i) {
3348  SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3349  SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3350  SDValue Shift = DAG.getConstant(
3351  HalfType.getScalarSizeInBits(), dl,
3352  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3353  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3354  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3355  }
3356  break;
3357  }
3358  break;
3359  }
3360  case ISD::MUL: {
3361  EVT VT = Node->getValueType(0);
3362  SDVTList VTs = DAG.getVTList(VT, VT);
3363  // See if multiply or divide can be lowered using two-result operations.
3364  // We just need the low half of the multiply; try both the signed
3365  // and unsigned forms. If the target supports both SMUL_LOHI and
3366  // UMUL_LOHI, form a preference by checking which forms of plain
3367  // MULH it supports.
3368  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3369  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3370  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3371  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3372  unsigned OpToUse = 0;
3373  if (HasSMUL_LOHI && !HasMULHS) {
3374  OpToUse = ISD::SMUL_LOHI;
3375  } else if (HasUMUL_LOHI && !HasMULHU) {
3376  OpToUse = ISD::UMUL_LOHI;
3377  } else if (HasSMUL_LOHI) {
3378  OpToUse = ISD::SMUL_LOHI;
3379  } else if (HasUMUL_LOHI) {
3380  OpToUse = ISD::UMUL_LOHI;
3381  }
3382  if (OpToUse) {
3383  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3384  Node->getOperand(1)));
3385  break;
3386  }
3387 
3388  SDValue Lo, Hi;
3389  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3390  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3391  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3392  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3393  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3394  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3396  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3397  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3398  SDValue Shift =
3399  DAG.getConstant(HalfType.getSizeInBits(), dl,
3400  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3401  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3402  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3403  }
3404  break;
3405  }
3406  case ISD::FSHL:
3407  case ISD::FSHR:
3408  if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3409  Results.push_back(Expanded);
3410  break;
3411  case ISD::ROTL:
3412  case ISD::ROTR:
3413  if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3414  Results.push_back(Expanded);
3415  break;
3416  case ISD::SADDSAT:
3417  case ISD::UADDSAT:
3418  case ISD::SSUBSAT:
3419  case ISD::USUBSAT:
3420  Results.push_back(TLI.expandAddSubSat(Node, DAG));
3421  break;
3422  case ISD::SSHLSAT:
3423  case ISD::USHLSAT:
3424  Results.push_back(TLI.expandShlSat(Node, DAG));
3425  break;
3426  case ISD::SMULFIX:
3427  case ISD::SMULFIXSAT:
3428  case ISD::UMULFIX:
3429  case ISD::UMULFIXSAT:
3430  Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3431  break;
3432  case ISD::SDIVFIX:
3433  case ISD::SDIVFIXSAT:
3434  case ISD::UDIVFIX:
3435  case ISD::UDIVFIXSAT:
3436  if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3437  Node->getOperand(0),
3438  Node->getOperand(1),
3439  Node->getConstantOperandVal(2),
3440  DAG)) {
3441  Results.push_back(V);
3442  break;
3443  }
3444  // FIXME: We might want to retry here with a wider type if we fail, if that
3445  // type is legal.
3446  // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3447  // <= 128 (which is the case for all of the default Embedded-C types),
3448  // we will only get here with types and scales that we could always expand
3449  // if we were allowed to generate libcalls to division functions of illegal
3450  // type. But we cannot do that.
3451  llvm_unreachable("Cannot expand DIVFIX!");
3452  case ISD::ADDCARRY:
3453  case ISD::SUBCARRY: {
3454  SDValue LHS = Node->getOperand(0);
3455  SDValue RHS = Node->getOperand(1);
3456  SDValue Carry = Node->getOperand(2);
3457 
3458  bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3459 
3460  // Initial add of the 2 operands.
3461  unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3462  EVT VT = LHS.getValueType();
3463  SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3464 
3465  // Initial check for overflow.
3466  EVT CarryType = Node->getValueType(1);
3467  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3469  SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3470 
3471  // Add of the sum and the carry.
3472  SDValue One = DAG.getConstant(1, dl, VT);
3473  SDValue CarryExt =
3474  DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3475  SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3476 
3477  // Second check for overflow. If we are adding, we can only overflow if the
3478  // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3479  // If we are subtracting, we can only overflow if the initial sum is 0 and
3480  // the carry is set, resulting in a new sum of all 1s.
3481  SDValue Zero = DAG.getConstant(0, dl, VT);
3482  SDValue Overflow2 =
3483  IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3484  : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3485  Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3486  DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3487 
3488  SDValue ResultCarry =
3489  DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3490 
3491  Results.push_back(Sum2);
3492  Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3493  break;
3494  }
3495  case ISD::SADDO:
3496  case ISD::SSUBO: {
3497  SDValue Result, Overflow;
3498  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3499  Results.push_back(Result);
3500  Results.push_back(Overflow);
3501  break;
3502  }
3503  case ISD::UADDO:
3504  case ISD::USUBO: {
3505  SDValue Result, Overflow;
3506  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3507  Results.push_back(Result);
3508  Results.push_back(Overflow);
3509  break;
3510  }
3511  case ISD::UMULO:
3512  case ISD::SMULO: {
3513  SDValue Result, Overflow;
3514  if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3515  Results.push_back(Result);
3516  Results.push_back(Overflow);
3517  }
3518  break;
3519  }
3520  case ISD::BUILD_PAIR: {
3521  EVT PairTy = Node->getValueType(0);
3522  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3523  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3524  Tmp2 = DAG.getNode(
3525  ISD::SHL, dl, PairTy, Tmp2,
3526  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3527  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3528  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3529  break;
3530  }
3531  case ISD::SELECT:
3532  Tmp1 = Node->getOperand(0);
3533  Tmp2 = Node->getOperand(1);
3534  Tmp3 = Node->getOperand(2);
3535  if (Tmp1.getOpcode() == ISD::SETCC) {
3536  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3537  Tmp2, Tmp3,
3538  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3539  } else {
3540  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3541  DAG.getConstant(0, dl, Tmp1.getValueType()),
3542  Tmp2, Tmp3, ISD::SETNE);
3543  }
3544  Tmp1->setFlags(Node->getFlags());
3545  Results.push_back(Tmp1);
3546  break;
3547  case ISD::BR_JT: {
3548  SDValue Chain = Node->getOperand(0);
3549  SDValue Table = Node->getOperand(1);
3550  SDValue Index = Node->getOperand(2);
3551 
3552  const DataLayout &TD = DAG.getDataLayout();
3553  EVT PTy = TLI.getPointerTy(TD);
3554 
3555  unsigned EntrySize =
3556  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3557 
3558  // For power-of-two jumptable entry sizes convert multiplication to a shift.
3559  // This transformation needs to be done here since otherwise the MIPS
3560  // backend will end up emitting a three instruction multiply sequence
3561  // instead of a single shift and MSP430 will call a runtime function.
3562  if (llvm::isPowerOf2_32(EntrySize))
3563  Index = DAG.getNode(
3564  ISD::SHL, dl, Index.getValueType(), Index,
3565  DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3566  else
3567  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3568  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3569  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3570  Index, Table);
3571 
3572  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3573  SDValue LD = DAG.getExtLoad(
3574  ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3575  MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3576  Addr = LD;
3577  if (TLI.isJumpTableRelative()) {
3578  // For PIC, the sequence is:
3579  // BRIND(load(Jumptable + index) + RelocBase)
3580  // RelocBase can be JumpTable, GOT or some sort of global base.
3581  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3582  TLI.getPICJumpTableRelocBase(Table, DAG));
3583  }
3584 
3585  Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3586  Results.push_back(Tmp1);
3587  break;
3588  }
3589  case ISD::BRCOND:
3590  // Expand brcond's setcc into its constituent parts and create a BR_CC
3591  // Node.
3592  Tmp1 = Node->getOperand(0);
3593  Tmp2 = Node->getOperand(1);
3594  if (Tmp2.getOpcode() == ISD::SETCC &&
3595  TLI.isOperationLegalOrCustom(ISD::BR_CC,
3596  Tmp2.getOperand(0).getValueType())) {
3597  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3598  Tmp2.getOperand(0), Tmp2.getOperand(1),
3599  Node->getOperand(2));
3600  } else {
3601  // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3602  if (Tmp2.isUndef() ||
3603  (Tmp2.getOpcode() == ISD::AND &&
3604  isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3605  cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3606  Tmp3 = Tmp2;
3607  else
3608  Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3609  DAG.getConstant(1, dl, Tmp2.getValueType()));
3610  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3611  DAG.getCondCode(ISD::SETNE), Tmp3,
3612  DAG.getConstant(0, dl, Tmp3.getValueType()),
3613  Node->getOperand(2));
3614  }
3615  Results.push_back(Tmp1);
3616  break;
3617  case ISD::SETCC:
3618  case ISD::VP_SETCC:
3619  case ISD::STRICT_FSETCC:
3620  case ISD::STRICT_FSETCCS: {
3621  bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
3622  bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
3623  Node->getOpcode() == ISD::STRICT_FSETCCS;
3624  bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
3625  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
3626  unsigned Offset = IsStrict ? 1 : 0;
3627  Tmp1 = Node->getOperand(0 + Offset);
3628  Tmp2 = Node->getOperand(1 + Offset);
3629  Tmp3 = Node->getOperand(2 + Offset);
3630  SDValue Mask, EVL;
3631  if (IsVP) {
3632  Mask = Node->getOperand(3 + Offset);
3633  EVL = Node->getOperand(4 + Offset);
3634  }
3635  bool Legalized = TLI.LegalizeSetCCCondCode(
3636  DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
3637  Chain, IsSignaling);
3638 
3639  if (Legalized) {
3640  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3641  // condition code, create a new SETCC node.
3642  if (Tmp3.getNode()) {
3643  if (IsStrict) {
3644  Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
3645  {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
3646  Chain = Tmp1.getValue(1);
3647  } else if (IsVP) {
3648  Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
3649  {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
3650  } else {
3651  Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
3652  Tmp2, Tmp3, Node->getFlags());
3653  }
3654  }
3655 
3656  // If we expanded the SETCC by inverting the condition code, then wrap
3657  // the existing SETCC in a NOT to restore the intended condition.
3658  if (NeedInvert) {
3659  if (!IsVP)
3660  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3661  else
3662  Tmp1 =
3663  DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
3664  }
3665 
3666  Results.push_back(Tmp1);
3667  if (IsStrict)
3668  Results.push_back(Chain);
3669 
3670  break;
3671  }
3672 
3673  // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
3674  // understand if this code is useful for strict nodes.
3675  assert(!IsStrict && "Don't know how to expand for strict nodes.");
3676 
3677  // Otherwise, SETCC for the given comparison type must be completely
3678  // illegal; expand it into a SELECT_CC.
3679  // FIXME: This drops the mask/evl for VP_SETCC.
3680  EVT VT = Node->getValueType(0);
3681  EVT Tmp1VT = Tmp1.getValueType();
3682  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3683  DAG.getBoolConstant(true, dl, VT, Tmp1VT),
3684  DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
3685  Tmp1->setFlags(Node->getFlags());
3686  Results.push_back(Tmp1);
3687  break;
3688  }
3689  case ISD::SELECT_CC: {
3690  // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
3691  Tmp1 = Node->getOperand(0); // LHS
3692  Tmp2 = Node->getOperand(1); // RHS
3693  Tmp3 = Node->getOperand(2); // True
3694  Tmp4 = Node->getOperand(3); // False
3695  EVT VT = Node->getValueType(0);
3696  SDValue Chain;
3697  SDValue CC = Node->getOperand(4);
3698  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3699 
3700  if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3701  // If the condition code is legal, then we need to expand this
3702  // node using SETCC and SELECT.
3703  EVT CmpVT = Tmp1.getValueType();
3704  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3705  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3706  "expanded.");
3707  EVT CCVT = getSetCCResultType(CmpVT);
3708  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3709  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3710  break;
3711  }
3712 
3713  // SELECT_CC is legal, so the condition code must not be.
3714  bool Legalized = false;
3715  // Try to legalize by inverting the condition. This is for targets that
3716  // might support an ordered version of a condition, but not the unordered
3717  // version (or vice versa).
3718  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
3719  if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3720  // Use the new condition code and swap true and false
3721  Legalized = true;
3722  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3723  Tmp1->setFlags(Node->getFlags());
3724  } else {
3725  // If The inverse is not legal, then try to swap the arguments using
3726  // the inverse condition code.
3727  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3728  if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3729  // The swapped inverse condition is legal, so swap true and false,
3730  // lhs and rhs.
3731  Legalized = true;
3732  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3733  Tmp1->setFlags(Node->getFlags());
3734  }
3735  }
3736 
3737  if (!Legalized) {
3738  Legalized = TLI.LegalizeSetCCCondCode(
3739  DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
3740  /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
3741 
3742  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3743 
3744  // If we expanded the SETCC by inverting the condition code, then swap
3745  // the True/False operands to match.
3746  if (NeedInvert)
3747  std::swap(Tmp3, Tmp4);
3748 
3749  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3750  // condition code, create a new SELECT_CC node.
3751  if (CC.getNode()) {
3752  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3753  Tmp1, Tmp2, Tmp3, Tmp4, CC);
3754  } else {
3755  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3756  CC = DAG.getCondCode(ISD::SETNE);
3757  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3758  Tmp2, Tmp3, Tmp4, CC);
3759  }
3760  Tmp1->setFlags(Node->getFlags());
3761  }
3762  Results.push_back(Tmp1);
3763  break;
3764  }
3765  case ISD::BR_CC: {
3766  // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
3767  SDValue Chain;
3768  Tmp1 = Node->getOperand(0); // Chain
3769  Tmp2 = Node->getOperand(2); // LHS
3770  Tmp3 = Node->getOperand(3); // RHS
3771  Tmp4 = Node->getOperand(1); // CC
3772 
3773  bool Legalized = TLI.LegalizeSetCCCondCode(
3774  DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
3775  /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
3776  (void)Legalized;
3777  assert(Legalized && "Can't legalize BR_CC with legal condition!");
3778 
3779  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3780  // node.
3781  if (Tmp4.getNode()) {
3782  assert(!NeedInvert && "Don't know how to invert BR_CC!");
3783 
3784  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3785  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3786  } else {
3787  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3788  Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
3789  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3790  Tmp2, Tmp3, Node->getOperand(4));
3791  }
3792  Results.push_back(Tmp1);
3793  break;
3794  }
3795  case ISD::BUILD_VECTOR:
3796  Results.push_back(ExpandBUILD_VECTOR(Node));
3797  break;
3798  case ISD::SPLAT_VECTOR:
3799  Results.push_back(ExpandSPLAT_VECTOR(Node));
3800  break;
3801  case ISD::SRA:
3802  case ISD::SRL:
3803  case ISD::SHL: {
3804  // Scalarize vector SRA/SRL/SHL.
3805  EVT VT = Node->getValueType(0);
3806  assert(VT.isVector() && "Unable to legalize non-vector shift");
3807  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3808  unsigned NumElem = VT.getVectorNumElements();
3809 
3810  SmallVector<SDValue, 8> Scalars;
3811  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3812  SDValue Ex =
3814  Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
3815  SDValue Sh =
3817  Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
3818  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3819  VT.getScalarType(), Ex, Sh));
3820  }
3821 
3822  SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3823  Results.push_back(Result);
3824  break;
3825  }
3826  case ISD::VECREDUCE_FADD:
3827  case ISD::VECREDUCE_FMUL:
3828  case ISD::VECREDUCE_ADD:
3829  case ISD::VECREDUCE_MUL:
3830  case ISD::VECREDUCE_AND:
3831  case ISD::VECREDUCE_OR:
3832  case ISD::VECREDUCE_XOR:
3833  case ISD::VECREDUCE_SMAX:
3834  case ISD::VECREDUCE_SMIN:
3835  case ISD::VECREDUCE_UMAX:
3836  case ISD::VECREDUCE_UMIN:
3837  case ISD::VECREDUCE_FMAX:
3838  case ISD::VECREDUCE_FMIN:
3839  Results.push_back(TLI.expandVecReduce(Node, DAG));
3840  break;
3842  case ISD::GlobalAddress:
3843  case ISD::GlobalTLSAddress:
3844  case ISD::ExternalSymbol:
3845  case ISD::ConstantPool:
3846  case ISD::JumpTable:
3849  case ISD::INTRINSIC_VOID:
3850  // FIXME: Custom lowering for these operations shouldn't return null!
3851  // Return true so that we don't call ConvertNodeToLibcall which also won't
3852  // do anything.
3853  return true;
3854  }
3855 
3856  if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
3857  // FIXME: We were asked to expand a strict floating-point operation,
3858  // but there is currently no expansion implemented that would preserve
3859  // the "strict" properties. For now, we just fall back to the non-strict
3860  // version if that is legal on the target. The actual mutation of the
3861  // operation will happen in SelectionDAGISel::DoInstructionSelection.
3862  switch (Node->getOpcode()) {
3863  default:
3864  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3865  Node->getValueType(0))
3867  return true;
3868  break;
3869  case ISD::STRICT_FSUB: {
3870  if (TLI.getStrictFPOperationAction(
3871  ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
3872  return true;
3873  if (TLI.getStrictFPOperationAction(
3874  ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
3875  break;
3876 
3877  EVT VT = Node->getValueType(0);
3878  const SDNodeFlags Flags = Node->getFlags();
3879  SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
3880  SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
3881  {Node->getOperand(0), Node->getOperand(1), Neg},
3882  Flags);
3883 
3884  Results.push_back(Fadd);
3885  Results.push_back(Fadd.getValue(1));
3886  break;
3887  }
3890  case ISD::STRICT_LRINT:
3891  case ISD::STRICT_LLRINT:
3892  case ISD::STRICT_LROUND:
3893  case ISD::STRICT_LLROUND:
3894  // These are registered by the operand type instead of the value
3895  // type. Reflect that here.
3896  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3897  Node->getOperand(1).getValueType())
3899  return true;
3900  break;
3901  }
3902  }
3903 
3904  // Replace the original node with the legalized result.
3905  if (Results.empty()) {
3906  LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3907  return false;
3908  }
3909 
3910  LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3911  ReplaceNode(Node, Results.data());
3912  return true;
3913 }
3914 
3915 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3916  LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3918  SDLoc dl(Node);
3919  // FIXME: Check flags on the node to see if we can use a finite call.
3920  unsigned Opc = Node->getOpcode();
3921  switch (Opc) {
3922  case ISD::ATOMIC_FENCE: {
3923  // If the target didn't lower this, lower it to '__sync_synchronize()' call
3924  // FIXME: handle "fence singlethread" more efficiently.
3926 
3928  CLI.setDebugLoc(dl)
3929  .setChain(Node->getOperand(0))
3930  .setLibCallee(
3931  CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3932  DAG.getExternalSymbol("__sync_synchronize",
3933  TLI.getPointerTy(DAG.getDataLayout())),
3934  std::move(Args));
3935 
3936  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3937 
3938  Results.push_back(CallResult.second);
3939  break;
3940  }
3941  // By default, atomic intrinsics are marked Legal and lowered. Targets
3942  // which don't support them directly, however, may want libcalls, in which
3943  // case they mark them Expand, and we get here.
3944  case ISD::ATOMIC_SWAP:
3945  case ISD::ATOMIC_LOAD_ADD:
3946  case ISD::ATOMIC_LOAD_SUB:
3947  case ISD::ATOMIC_LOAD_AND:
3948  case ISD::ATOMIC_LOAD_CLR:
3949  case ISD::ATOMIC_LOAD_OR:
3950  case ISD::ATOMIC_LOAD_XOR:
3951  case ISD::ATOMIC_LOAD_NAND:
3952  case ISD::ATOMIC_LOAD_MIN:
3953  case ISD::ATOMIC_LOAD_MAX:
3954  case ISD::ATOMIC_LOAD_UMIN:
3955  case ISD::ATOMIC_LOAD_UMAX:
3956  case ISD::ATOMIC_CMP_SWAP: {
3957  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3958  AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3959  RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
3960  EVT RetVT = Node->getValueType(0);
3963  if (TLI.getLibcallName(LC)) {
3964  // If outline atomic available, prepare its arguments and expand.
3965  Ops.append(Node->op_begin() + 2, Node->op_end());
3966  Ops.push_back(Node->getOperand(1));
3967 
3968  } else {
3969  LC = RTLIB::getSYNC(Opc, VT);
3970  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3971  "Unexpected atomic op or value type!");
3972  // Arguments for expansion to sync libcall
3973  Ops.append(Node->op_begin() + 1, Node->op_end());
3974  }
3975  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3976  Ops, CallOptions,
3977  SDLoc(Node),
3978  Node->getOperand(0));
3979  Results.push_back(Tmp.first);
3980  Results.push_back(Tmp.second);
3981  break;
3982  }
3983  case ISD::TRAP: {
3984  // If this operation is not supported, lower it to 'abort()' call
3987  CLI.setDebugLoc(dl)
3988  .setChain(Node->getOperand(0))
3989  .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3990  DAG.getExternalSymbol(
3991  "abort", TLI.getPointerTy(DAG.getDataLayout())),
3992  std::move(Args));
3993  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3994 
3995  Results.push_back(CallResult.second);
3996  break;
3997  }
3998  case ISD::FMINNUM:
3999  case ISD::STRICT_FMINNUM:
4000  ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4001  RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4002  RTLIB::FMIN_PPCF128, Results);
4003  break;
4004  case ISD::FMAXNUM:
4005  case ISD::STRICT_FMAXNUM:
4006  ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4007  RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4008  RTLIB::FMAX_PPCF128, Results);
4009  break;
4010  case ISD::FSQRT:
4011  case ISD::STRICT_FSQRT:
4012  ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4013  RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4014  RTLIB::SQRT_PPCF128, Results);
4015  break;
4016  case ISD::FCBRT:
4017  ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4018  RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4019  RTLIB::CBRT_PPCF128, Results);
4020  break;
4021  case ISD::FSIN:
4022  case ISD::STRICT_FSIN:
4023  ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4024  RTLIB::SIN_F80, RTLIB::SIN_F128,
4025  RTLIB::SIN_PPCF128, Results);
4026  break;
4027  case ISD::FCOS:
4028  case ISD::STRICT_FCOS:
4029  ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4030  RTLIB::COS_F80, RTLIB::COS_F128,
4031  RTLIB::COS_PPCF128, Results);
4032  break;
4033  case ISD::FSINCOS:
4034  // Expand into sincos libcall.
4035  ExpandSinCosLibCall(Node, Results);
4036  break;
4037  case ISD::FLOG:
4038  case ISD::STRICT_FLOG:
4039  ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4040  RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4041  break;
4042  case ISD::FLOG2:
4043  case ISD::STRICT_FLOG2:
4044  ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4045  RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4046  break;
4047  case ISD::FLOG10:
4048  case ISD::STRICT_FLOG10:
4049  ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4050  RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4051  break;
4052  case ISD::FEXP:
4053  case ISD::STRICT_FEXP:
4054  ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4055  RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4056  break;
4057  case ISD::FEXP2:
4058  case ISD::STRICT_FEXP2:
4059  ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4060  RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4061  break;
4062  case ISD::FTRUNC:
4063  case ISD::STRICT_FTRUNC:
4064  ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4065  RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4066  RTLIB::TRUNC_PPCF128, Results);
4067  break;
4068  case ISD::FFLOOR:
4069  case ISD::STRICT_FFLOOR:
4070  ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4071  RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4072  RTLIB::FLOOR_PPCF128, Results);
4073  break;
4074  case ISD::FCEIL:
4075  case ISD::STRICT_FCEIL:
4076  ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4077  RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4078  RTLIB::CEIL_PPCF128, Results);
4079  break;
4080  case ISD::FRINT:
4081  case ISD::STRICT_FRINT:
4082  ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4083  RTLIB::RINT_F80, RTLIB::RINT_F128,
4084  RTLIB::RINT_PPCF128, Results);
4085  break;
4086  case ISD::FNEARBYINT:
4088  ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4089  RTLIB::NEARBYINT_F64,
4090  RTLIB::NEARBYINT_F80,
4091  RTLIB::NEARBYINT_F128,
4092  RTLIB::NEARBYINT_PPCF128, Results);
4093  break;
4094  case ISD::FROUND:
4095  case ISD::STRICT_FROUND:
4096  ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4097  RTLIB::ROUND_F64,
4098  RTLIB::ROUND_F80,
4099  RTLIB::ROUND_F128,
4100  RTLIB::ROUND_PPCF128, Results);
4101  break;
4102  case ISD::FROUNDEVEN:
4104  ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4105  RTLIB::ROUNDEVEN_F64,
4106  RTLIB::ROUNDEVEN_F80,
4107  RTLIB::ROUNDEVEN_F128,
4108  RTLIB::ROUNDEVEN_PPCF128, Results);
4109  break;
4110  case ISD::FPOWI:
4111  case ISD::STRICT_FPOWI: {
4112  RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4113  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4114  if (!TLI.getLibcallName(LC)) {
4115  // Some targets don't have a powi libcall; use pow instead.
4116  if (Node->isStrictFPOpcode()) {
4117  SDValue Exponent =
4118  DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4119  {Node->getValueType(0), Node->getValueType(1)},
4120  {Node->getOperand(0), Node->getOperand(2)});
4121  SDValue FPOW =
4122  DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4123  {Node->getValueType(0), Node->getValueType(1)},
4124  {Exponent.getValue(1), Node->getOperand(1), Exponent});
4125  Results.push_back(FPOW);
4126  Results.push_back(FPOW.getValue(1));
4127  } else {
4128  SDValue Exponent =
4129  DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4130  Node->getOperand(1));
4131  Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4132  Node->getValueType(0),
4133  Node->getOperand(0), Exponent));
4134  }
4135  break;
4136  }
4137  unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4138  bool ExponentHasSizeOfInt =
4139  DAG.getLibInfo().getIntSize() ==
4140  Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4141  if (!ExponentHasSizeOfInt) {
4142  // If the exponent does not match with sizeof(int) a libcall to
4143  // RTLIB::POWI would use the wrong type for the argument.
4144  DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4145  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4146  break;
4147  }
4148  ExpandFPLibCall(Node, LC, Results);
4149  break;
4150  }
4151  case ISD::FPOW:
4152  case ISD::STRICT_FPOW:
4153  ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4154  RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4155  break;
4156  case ISD::LROUND:
4157  case ISD::STRICT_LROUND:
4158  ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4159  RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4160  RTLIB::LROUND_F128,
4161  RTLIB::LROUND_PPCF128, Results);
4162  break;
4163  case ISD::LLROUND:
4164  case ISD::STRICT_LLROUND:
4165  ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4166  RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4167  RTLIB::LLROUND_F128,
4168  RTLIB::LLROUND_PPCF128, Results);
4169  break;
4170  case ISD::LRINT:
4171  case ISD::STRICT_LRINT:
4172  ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4173  RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4174  RTLIB::LRINT_F128,
4175  RTLIB::LRINT_PPCF128, Results);
4176  break;
4177  case ISD::LLRINT:
4178  case ISD::STRICT_LLRINT:
4179  ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4180  RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4181  RTLIB::LLRINT_F128,
4182  RTLIB::LLRINT_PPCF128, Results);
4183  break;
4184  case ISD::FDIV:
4185  case ISD::STRICT_FDIV:
4186  ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4187  RTLIB::DIV_F80, RTLIB::DIV_F128,
4188  RTLIB::DIV_PPCF128, Results);
4189  break;
4190  case ISD::FREM:
4191  case ISD::STRICT_FREM:
4192  ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4193  RTLIB::REM_F80, RTLIB::REM_F128,
4194  RTLIB::REM_PPCF128, Results);
4195  break;
4196  case ISD::FMA:
4197  case ISD::STRICT_FMA:
4198  ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4199  RTLIB::FMA_F80, RTLIB::FMA_F128,
4200  RTLIB::FMA_PPCF128, Results);
4201  break;
4202  case ISD::FADD:
4203  case ISD::STRICT_FADD:
4204  ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4205  RTLIB::ADD_F80, RTLIB::ADD_F128,
4206  RTLIB::ADD_PPCF128, Results);
4207  break;
4208  case ISD::FMUL:
4209  case ISD::STRICT_FMUL:
4210  ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4211  RTLIB::MUL_F80, RTLIB::MUL_F128,
4212  RTLIB::MUL_PPCF128, Results);
4213  break;
4214  case ISD::FP16_TO_FP:
4215  if (Node->getValueType(0) == MVT::f32) {
4216  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4217  }
4218  break;
4219  case ISD::STRICT_FP16_TO_FP: {
4220  if (Node->getValueType(0) == MVT::f32) {
4222  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4223  DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4224  SDLoc(Node), Node->getOperand(0));
4225  Results.push_back(Tmp.first);
4226  Results.push_back(Tmp.second);
4227  }
4228  break;
4229  }
4230  case ISD::FP_TO_FP16: {
4231  RTLIB::Libcall LC =
4232  RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4233  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4234  Results.push_back(ExpandLibCall(LC, Node, false));
4235  break;
4236  }
4237  case ISD::FP_TO_BF16: {
4238  RTLIB::Libcall LC =
4239  RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4240  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4241  Results.push_back(ExpandLibCall(LC, Node, false));
4242  break;
4243  }
4246  case ISD::SINT_TO_FP:
4247  case ISD::UINT_TO_FP: {
4248  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4249  bool IsStrict = Node->isStrictFPOpcode();
4250  bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4251  Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4252  EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4253  EVT RVT = Node->getValueType(0);
4254  EVT NVT = EVT();
4255  SDLoc dl(Node);
4256 
4257  // Even if the input is legal, no libcall may exactly match, eg. we don't
4258  // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4259  // eg: i13 -> fp. Then, look for an appropriate libcall.
4260  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4261  for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4262  t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4263  ++t) {
4264  NVT = (MVT::SimpleValueType)t;
4265  // The source needs to big enough to hold the operand.
4266  if (NVT.bitsGE(SVT))
4267  LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4268  : RTLIB::getUINTTOFP(NVT, RVT);
4269  }
4270  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4271 
4272  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4273  // Sign/zero extend the argument if the libcall takes a larger type.
4274  SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4275  NVT, Node->getOperand(IsStrict ? 1 : 0));
4277  CallOptions.setSExt(Signed);
4278  std::pair<SDValue, SDValue> Tmp =
4279  TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4280  Results.push_back(Tmp.first);
4281  if (IsStrict)
4282  Results.push_back(Tmp.second);
4283  break;
4284  }
4285  case ISD::FP_TO_SINT:
4286  case ISD::FP_TO_UINT:
4288  case ISD::STRICT_FP_TO_UINT: {
4289  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4290  bool IsStrict = Node->isStrictFPOpcode();
4291  bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4292  Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4293 
4294  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4295  EVT SVT = Op.getValueType();
4296  EVT RVT = Node->getValueType(0);
4297  EVT NVT = EVT();
4298  SDLoc dl(Node);
4299 
4300  // Even if the result is legal, no libcall may exactly match, eg. we don't
4301  // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4302  // eg: fp -> i32. Then, look for an appropriate libcall.
4303  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4304  for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4305  IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4306  ++IntVT) {
4307  NVT = (MVT::SimpleValueType)IntVT;
4308  // The type needs to big enough to hold the result.
4309  if (NVT.bitsGE(RVT))
4310  LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4311  : RTLIB::getFPTOUINT(SVT, NVT);
4312  }
4313  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4314 
4315  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4317  std::pair<SDValue, SDValue> Tmp =
4318  TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4319 
4320  // Truncate the result if the libcall returns a larger type.
4321  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4322  if (IsStrict)
4323  Results.push_back(Tmp.second);
4324  break;
4325  }
4326 
4327  case ISD::FP_ROUND:
4328  case ISD::STRICT_FP_ROUND: {
4329  // X = FP_ROUND(Y, TRUNC)
4330  // TRUNC is a flag, which is always an integer that is zero or one.
4331  // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4332  // is known to not change the value of Y.
4333  // We can only expand it into libcall if the TRUNC is 0.
4334  bool IsStrict = Node->isStrictFPOpcode();
4335  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4336  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4337  EVT VT = Node->getValueType(0);
4338  assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4339  "Unable to expand as libcall if it is not normal rounding");
4340 
4341  RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4342  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4343 
4345  std::pair<SDValue, SDValue> Tmp =
4346  TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4347  Results.push_back(Tmp.first);
4348  if (IsStrict)
4349  Results.push_back(Tmp.second);
4350  break;
4351  }
4352  case ISD::FP_EXTEND: {
4353  Results.push_back(
4354  ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4355  Node->getValueType(0)),
4356  Node, false));
4357  break;
4358  }
4359  case ISD::STRICT_FP_EXTEND:
4360  case ISD::STRICT_FP_TO_FP16: {
4361  RTLIB::Libcall LC =
4362  Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4363  ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4364  : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4365  Node->getValueType(0));
4366  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4367 
4369  std::pair<SDValue, SDValue> Tmp =
4370  TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4371  CallOptions, SDLoc(Node), Node->getOperand(0));
4372  Results.push_back(Tmp.first);
4373  Results.push_back(Tmp.second);
4374  break;
4375  }
4376  case ISD::FSUB:
4377  case ISD::STRICT_FSUB:
4378  ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4379  RTLIB::SUB_F80, RTLIB::SUB_F128,
4380  RTLIB::SUB_PPCF128, Results);
4381  break;
4382  case ISD::SREM:
4383  Results.push_back(ExpandIntLibCall(Node, true,
4384  RTLIB::SREM_I8,
4385  RTLIB::SREM_I16, RTLIB::SREM_I32,
4386  RTLIB::SREM_I64, RTLIB::SREM_I128));
4387  break;
4388  case ISD::UREM:
4389  Results.push_back(ExpandIntLibCall(Node, false,
4390  RTLIB::UREM_I8,
4391  RTLIB::UREM_I16, RTLIB::UREM_I32,
4392  RTLIB::UREM_I64, RTLIB::UREM_I128));
4393  break;
4394  case ISD::SDIV:
4395  Results.push_back(ExpandIntLibCall(Node, true,
4396  RTLIB::SDIV_I8,
4397  RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4398  RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4399  break;
4400  case ISD::UDIV:
4401  Results.push_back(ExpandIntLibCall(Node, false,
4402  RTLIB::UDIV_I8,
4403  RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4404  RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4405  break;
4406  case ISD::SDIVREM:
4407  case ISD::UDIVREM:
4408  // Expand into divrem libcall
4409  ExpandDivRemLibCall(Node, Results);
4410  break;
4411  case ISD::MUL:
4412  Results.push_back(ExpandIntLibCall(Node, false,
4413  RTLIB::MUL_I8,
4414  RTLIB::MUL_I16, RTLIB::MUL_I32,
4415  RTLIB::MUL_I64, RTLIB::MUL_I128));
4416  break;
4417  case ISD::CTLZ_ZERO_UNDEF:
4418  switch (Node->getSimpleValueType(0).SimpleTy) {
4419  default:
4420  llvm_unreachable("LibCall explicitly requested, but not available");
4421  case MVT::i32:
4422  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4423  break;
4424  case MVT::i64:
4425  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4426  break;
4427  case MVT::i128:
4428  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4429  break;
4430  }
4431  break;
4432  }
4433 
4434  // Replace the original node with the legalized result.
4435  if (!Results.empty()) {
4436  LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4437  ReplaceNode(Node, Results.data());
4438  } else
4439  LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4440 }
4441 
4442 // Determine the vector type to use in place of an original scalar element when
4443 // promoting equally sized vectors.
4445  MVT EltVT, MVT NewEltVT) {
4446  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4447  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4448  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4449  return MidVT;
4450 }
4451 
4452 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4453  LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4455  MVT OVT = Node->getSimpleValueType(0);
4456  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4457  Node->getOpcode() == ISD::SINT_TO_FP ||
4458  Node->getOpcode() == ISD::SETCC ||
4459  Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4460  Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4461  OVT = Node->getOperand(0).getSimpleValueType();
4462  }
4463  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4464  Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4465  Node->getOpcode() == ISD::STRICT_FSETCC ||
4466  Node->getOpcode() == ISD::STRICT_FSETCCS)
4467  OVT = Node->getOperand(1).getSimpleValueType();
4468  if (Node->getOpcode() == ISD::BR_CC ||
4469  Node->getOpcode() == ISD::SELECT_CC)
4470  OVT = Node->getOperand(2).getSimpleValueType();
4471  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4472  SDLoc dl(Node);
4473  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4474  switch (Node->getOpcode()) {
4475  case ISD::CTTZ:
4476  case ISD::CTTZ_ZERO_UNDEF:
4477  case ISD::CTLZ:
4478  case ISD::CTLZ_ZERO_UNDEF:
4479  case ISD::CTPOP:
4480  // Zero extend the argument unless its cttz, then use any_extend.
4481  if (Node->getOpcode() == ISD::CTTZ ||
4482  Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4483  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4484  else
4485  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4486 
4487  if (Node->getOpcode() == ISD::CTTZ) {
4488  // The count is the same in the promoted type except if the original
4489  // value was zero. This can be handled by setting the bit just off
4490  // the top of the original type.
4491  auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4492  OVT.getSizeInBits());
4493  Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4494  DAG.getConstant(TopBit, dl, NVT));
4495  }
4496  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4497  // already the correct result.
4498  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4499  if (Node->getOpcode() == ISD::CTLZ ||
4500  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4501  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4502  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4503  DAG.getConstant(NVT.getSizeInBits() -
4504  OVT.getSizeInBits(), dl, NVT));
4505  }
4506  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4507  break;
4508  case ISD::BITREVERSE:
4509  case ISD::BSWAP: {
4510  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4511  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4512  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4513  Tmp1 = DAG.getNode(
4514  ISD::SRL, dl, NVT, Tmp1,
4515  DAG.getConstant(DiffBits, dl,
4516  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4517 
4518  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4519  break;
4520  }
4521  case ISD::FP_TO_UINT:
4523  case ISD::FP_TO_SINT:
4525  PromoteLegalFP_TO_INT(Node, dl, Results);
4526  break;
4527  case ISD::FP_TO_UINT_SAT:
4528  case ISD::FP_TO_SINT_SAT:
4529  Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
4530  break;
4531  case ISD::UINT_TO_FP:
4533  case ISD::SINT_TO_FP:
4535  PromoteLegalINT_TO_FP(Node, dl, Results);
4536  break;
4537  case ISD::VAARG: {
4538  SDValue Chain = Node->getOperand(0); // Get the chain.
4539  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4540 
4541  unsigned TruncOp;
4542  if (OVT.isVector()) {
4543  TruncOp = ISD::BITCAST;
4544  } else {
4545  assert(OVT.isInteger()
4546  && "VAARG promotion is supported only for vectors or integer types");
4547  TruncOp = ISD::TRUNCATE;
4548  }
4549 
4550  // Perform the larger operation, then convert back
4551  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4552  Node->getConstantOperandVal(3));
4553  Chain = Tmp1.getValue(1);
4554 
4555  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4556 
4557  // Modified the chain result - switch anything that used the old chain to
4558  // use the new one.
4559  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4560  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4561  if (UpdatedNodes) {
4562  UpdatedNodes->insert(Tmp2.getNode());
4563  UpdatedNodes->insert(Chain.getNode());
4564  }
4565  ReplacedNode(Node);
4566  break;
4567  }
4568  case ISD::MUL:
4569  case ISD::SDIV:
4570  case ISD::SREM:
4571  case ISD::UDIV:
4572  case ISD::UREM:
4573  case ISD::AND:
4574  case ISD::OR:
4575  case ISD::XOR: {
4576  unsigned ExtOp, TruncOp;
4577  if (OVT.isVector()) {
4578  ExtOp = ISD::BITCAST;
4579  TruncOp = ISD::BITCAST;
4580  } else {
4581  assert(OVT.isInteger() && "Cannot promote logic operation");
4582 
4583  switch (Node->getOpcode()) {
4584  default:
4585  ExtOp = ISD::ANY_EXTEND;
4586  break;
4587  case ISD::SDIV:
4588  case ISD::SREM:
4589  ExtOp = ISD::SIGN_EXTEND;
4590  break;
4591  case ISD::UDIV:
4592  case ISD::UREM:
4593  ExtOp = ISD::ZERO_EXTEND;
4594  break;
4595  }
4596  TruncOp = ISD::TRUNCATE;
4597  }
4598  // Promote each of the values to the new type.
4599  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4600  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4601  // Perform the larger operation, then convert back
4602  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4603  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4604  break;
4605  }
4606  case ISD::UMUL_LOHI:
4607  case ISD::SMUL_LOHI: {
4608  // Promote to a multiply in a wider integer type.
4609  unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4610  : ISD::SIGN_EXTEND;
4611  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4612  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4613  Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4614 
4615  auto &DL = DAG.getDataLayout();
4616  unsigned OriginalSize = OVT.getScalarSizeInBits();
4617  Tmp2 = DAG.getNode(
4618  ISD::SRL, dl, NVT, Tmp1,
4619  DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4620  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4621  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4622  break;
4623  }
4624  case ISD::SELECT: {
4625  unsigned ExtOp, TruncOp;
4626  if (Node->getValueType(0).isVector() ||
4627  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4628  ExtOp = ISD::BITCAST;
4629  TruncOp = ISD::BITCAST;
4630  } else if (Node->getValueType(0).isInteger()) {
4631  ExtOp = ISD::ANY_EXTEND;
4632  TruncOp = ISD::TRUNCATE;
4633  } else {
4634  ExtOp = ISD::FP_EXTEND;
4635  TruncOp = ISD::FP_ROUND;
4636  }
4637  Tmp1 = Node->getOperand(0);
4638  // Promote each of the values to the new type.
4639  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4640  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4641  // Perform the larger operation, then round down.
4642  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4643  Tmp1->setFlags(Node->getFlags());
4644  if (TruncOp != ISD::FP_ROUND)
4645  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4646  else
4647  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4648  DAG.getIntPtrConstant(0, dl));
4649  Results.push_back(Tmp1);
4650  break;
4651  }
4652  case ISD::VECTOR_SHUFFLE: {
4653  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4654 
4655  // Cast the two input vectors.
4656  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4657  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4658 
4659  // Convert the shuffle mask to the right # elements.
4660  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4661  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4662  Results.push_back(Tmp1);
4663  break;
4664  }
4665  case ISD::VECTOR_SPLICE: {
4666  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4667  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
4668  Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
4669  Node->getOperand(2));
4670  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
4671  break;
4672  }
4673  case ISD::SELECT_CC: {
4674  SDValue Cond = Node->getOperand(4);
4675  ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
4676  // Type of the comparison operands.
4677  MVT CVT = Node->getSimpleValueType(0);
4678  assert(CVT == OVT && "not handled");
4679 
4680  unsigned ExtOp = ISD::FP_EXTEND;
4681  if (NVT.isInteger()) {
4683  }
4684 
4685  // Promote the comparison operands, if needed.
4686  if (TLI.isCondCodeLegal(CCCode, CVT)) {
4687  Tmp1 = Node->getOperand(0);
4688  Tmp2 = Node->getOperand(1);
4689  } else {
4690  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4691  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4692  }
4693  // Cast the true/false operands.
4694  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4695  Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4696 
4697  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
4698  Node->getFlags());
4699 
4700  // Cast the result back to the original type.
4701  if (ExtOp != ISD::FP_EXTEND)
4702  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
4703  else
4704  Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
4705  DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
4706 
4707  Results.push_back(Tmp1);
4708  break;
4709  }
4710  case ISD::SETCC:
4711  case ISD::STRICT_FSETCC:
4712  case ISD::STRICT_FSETCCS: {
4713  unsigned ExtOp = ISD::FP_EXTEND;
4714  if (NVT.isInteger()) {
4715  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
4717  }
4718  if (Node->isStrictFPOpcode()) {
4719  SDValue InChain = Node->getOperand(0);
4720  std::tie(Tmp1, std::ignore) =
4721  DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
4722  std::tie(Tmp2, std::ignore) =
4723  DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
4724  SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
4725  SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
4726  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
4727  Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
4728  {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
4729  Node->getFlags()));
4730  Results.push_back(Results.back().getValue(1));
4731  break;
4732  }
4733  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4734  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4735  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4736  Tmp2, Node->getOperand(2), Node->getFlags()));
4737  break;
4738  }
4739  case ISD::BR_CC: {
4740  unsigned ExtOp = ISD::FP_EXTEND;
4741  if (NVT.isInteger()) {
4742  ISD::CondCode CCCode =
4743  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4745  }
4746  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4747  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4748  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4749  Node->getOperand(0), Node->getOperand(1),
4750  Tmp1, Tmp2, Node->getOperand(4)));
4751  break;
4752  }
4753  case ISD::FADD:
4754  case ISD::FSUB:
4755  case ISD::FMUL:
4756  case ISD::FDIV:
4757  case ISD::FREM:
4758  case ISD::FMINNUM:
4759  case ISD::FMAXNUM:
4760  case ISD::FPOW:
4761  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4762  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4763  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4764  Node->getFlags());
4765  Results.push_back(
4766  DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
4767  DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
4768  break;
4769  case ISD::STRICT_FADD:
4770  case ISD::STRICT_FSUB:
4771  case ISD::STRICT_FMUL:
4772  case ISD::STRICT_FDIV:
4773  case ISD::STRICT_FMINNUM:
4774  case ISD::STRICT_FMAXNUM:
4775  case ISD::STRICT_FREM:
4776  case ISD::STRICT_FPOW:
4777  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4778  {Node->getOperand(0), Node->getOperand(1)});
4779  Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4780  {Node->getOperand(0), Node->getOperand(2)});
4781  Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4782  Tmp2.getValue(1));
4783  Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4784  {Tmp3, Tmp1, Tmp2});
4785  Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4786  {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
4787  Results.push_back(Tmp1);
4788  Results.push_back(Tmp1.getValue(1));
4789  break;
4790  case ISD::FMA:
4791  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4792  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4793  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4794  Results.push_back(
4795  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4796  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4797  DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
4798  break;
4799  case ISD::STRICT_FMA:
4800  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4801  {Node->getOperand(0), Node->getOperand(1)});
4802  Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4803  {Node->getOperand(0), Node->getOperand(2)});
4804  Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4805  {Node->getOperand(0), Node->getOperand(3)});
4806  Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4807  Tmp2.getValue(1), Tmp3.getValue(1));
4808  Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4809  {Tmp4, Tmp1, Tmp2, Tmp3});
4810  Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4811  {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
4812  Results.push_back(Tmp4);
4813  Results.push_back(Tmp4.getValue(1));
4814  break;
4815  case ISD::FCOPYSIGN:
4816  case ISD::FPOWI: {
4817  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4818  Tmp2 = Node->getOperand(1);
4819  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4820 
4821  // fcopysign doesn't change anything but the sign bit, so
4822  // (fp_round (fcopysign (fpext a), b))
4823  // is as precise as
4824  // (fp_round (fpext a))
4825  // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4826  const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4827  Results.push_back(
4828  DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
4829  DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true)));
4830  break;
4831  }
4832  case ISD::STRICT_FPOWI:
4833  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4834  {Node->getOperand(0), Node->getOperand(1)});
4835  Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4836  {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
4837  Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4838  {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4839  Results.push_back(Tmp3);
4840  Results.push_back(Tmp3.getValue(1));
4841  break;
4842  case ISD::FFLOOR:
4843  case ISD::FCEIL:
4844  case ISD::FRINT:
4845  case ISD::FNEARBYINT:
4846  case ISD::FROUND:
4847  case ISD::FROUNDEVEN:
4848  case ISD::FTRUNC:
4849  case ISD::FNEG:
4850  case ISD::FSQRT:
4851  case ISD::FSIN:
4852  case ISD::FCOS:
4853  case ISD::FLOG:
4854  case ISD::FLOG2:
4855  case ISD::FLOG10:
4856  case ISD::FABS:
4857  case ISD::FEXP:
4858  case ISD::FEXP2:
4859  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4860  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4861  Results.push_back(
4862  DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
4863  DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
4864  break;
4865  case ISD::STRICT_FFLOOR:
4866  case ISD::STRICT_FCEIL:
4867  case ISD::STRICT_FRINT:
4869  case ISD::STRICT_FROUND:
4871  case ISD::STRICT_FTRUNC:
4872  case ISD::STRICT_FSQRT:
4873  case ISD::STRICT_FSIN:
4874  case ISD::STRICT_FCOS:
4875  case ISD::STRICT_FLOG:
4876  case ISD::STRICT_FLOG2:
4877  case ISD::STRICT_FLOG10:
4878  case ISD::STRICT_FEXP:
4879  case ISD::STRICT_FEXP2:
4880  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4881  {Node->getOperand(0), Node->getOperand(1)});
4882  Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4883  {Tmp1.getValue(1), Tmp1});
4884  Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4885  {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4886  Results.push_back(Tmp3);
4887  Results.push_back(Tmp3.getValue(1));
4888  break;
4889  case ISD::BUILD_VECTOR: {
4890  MVT EltVT = OVT.getVectorElementType();
4891  MVT NewEltVT = NVT.getVectorElementType();
4892 
4893  // Handle bitcasts to a different vector type with the same total bit size
4894  //
4895  // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4896  // =>
4897  // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4898 
4899  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4900  "Invalid promote type for build_vector");
4901  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4902 
4903  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4904 
4905  SmallVector<SDValue, 8> NewOps;
4906  for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4907  SDValue Op = Node->getOperand(I);
4908  NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4909  }
4910 
4911  SDLoc SL(Node);
4912  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4913  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4914  Results.push_back(CvtVec);
4915  break;
4916  }
4917  case ISD::EXTRACT_VECTOR_ELT: {
4918  MVT EltVT = OVT.getVectorElementType();
4919  MVT NewEltVT = NVT.getVectorElementType();
4920 
4921  // Handle bitcasts to a different vector type with the same total bit size.
4922  //
4923  // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4924  // =>
4925  // v4i32:castx = bitcast x:v2i64
4926  //
4927  // i64 = bitcast
4928  // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4929  // (i32 (extract_vector_elt castx, (2 * y + 1)))
4930  //
4931 
4932  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4933  "Invalid promote type for extract_vector_elt");
4934  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4935 
4936  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4937  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4938 
4939  SDValue Idx = Node->getOperand(1);
4940  EVT IdxVT = Idx.getValueType();
4941  SDLoc SL(Node);
4942  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4943  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4944 
4945  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4946 
4947  SmallVector<SDValue, 8> NewOps;
4948  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4949  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4950  SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4951 
4952  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4953  CastVec, TmpIdx);
4954  NewOps.push_back(Elt);
4955  }
4956 
4957  SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4958  Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4959  break;
4960  }
4961  case ISD::INSERT_VECTOR_ELT: {
4962  MVT EltVT = OVT.getVectorElementType();
4963  MVT NewEltVT = NVT.getVectorElementType();
4964 
4965  // Handle bitcasts to a different vector type with the same total bit size
4966  //
4967  // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4968  // =>
4969  // v4i32:castx = bitcast x:v2i64
4970  // v2i32:casty = bitcast y:i64
4971  //
4972  // v2i64 = bitcast
4973