LLVM  12.0.0git
LegalizeTypes.h
Go to the documentation of this file.
1 //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the DAGTypeLegalizer class. This is a private interface
10 // shared between the code that implements the SelectionDAG::LegalizeTypes
11 // method.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
16 #define LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
17 
18 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/Support/Compiler.h"
22 #include "llvm/Support/Debug.h"
23 
24 namespace llvm {
25 
26 //===----------------------------------------------------------------------===//
27 /// This takes an arbitrary SelectionDAG as input and hacks on it until only
28 /// value types the target machine can handle are left. This involves promoting
29 /// small sizes to large sizes or splitting up large values into small values.
30 ///
32  const TargetLowering &TLI;
33  SelectionDAG &DAG;
34 public:
35  /// This pass uses the NodeId on the SDNodes to hold information about the
36  /// state of the node. The enum has all the values.
37  enum NodeIdFlags {
38  /// All operands have been processed, so this node is ready to be handled.
39  ReadyToProcess = 0,
40 
41  /// This is a new node, not before seen, that was created in the process of
42  /// legalizing some other node.
43  NewNode = -1,
44 
45  /// This node's ID needs to be set to the number of its unprocessed
46  /// operands.
47  Unanalyzed = -2,
48 
49  /// This is a node that has already been processed.
50  Processed = -3
51 
52  // 1+ - This is a node which has this many unprocessed operands.
53  };
54 private:
55 
56  /// This is a bitvector that contains two bits for each simple value type,
57  /// where the two bits correspond to the LegalizeAction enum from
58  /// TargetLowering. This can be queried with "getTypeAction(VT)".
59  TargetLowering::ValueTypeActionImpl ValueTypeActions;
60 
61  /// Return how we should legalize values of this type.
62  TargetLowering::LegalizeTypeAction getTypeAction(EVT VT) const {
63  return TLI.getTypeAction(*DAG.getContext(), VT);
64  }
65 
66  /// Return true if this type is legal on this target.
67  bool isTypeLegal(EVT VT) const {
68  return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;
69  }
70 
71  /// Return true if this is a simple legal type.
72  bool isSimpleLegalType(EVT VT) const {
73  return VT.isSimple() && TLI.isTypeLegal(VT);
74  }
75 
76  EVT getSetCCResultType(EVT VT) const {
77  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
78  }
79 
80  /// Pretend all of this node's results are legal.
81  bool IgnoreNodeResults(SDNode *N) const {
82  return N->getOpcode() == ISD::TargetConstant ||
83  N->getOpcode() == ISD::Register;
84  }
85 
86  // Bijection from SDValue to unique id. As each created node gets a
87  // new id we do not need to worry about reuse expunging. Should we
88  // run out of ids, we can do a one time expensive compactifcation.
89  typedef unsigned TableId;
90 
91  TableId NextValueId = 1;
92 
95 
96  /// For integer nodes that are below legal width, this map indicates what
97  /// promoted value to use.
98  SmallDenseMap<TableId, TableId, 8> PromotedIntegers;
99 
100  /// For integer nodes that need to be expanded this map indicates which
101  /// operands are the expanded version of the input.
103 
104  /// For floating-point nodes converted to integers of the same size, this map
105  /// indicates the converted value to use.
106  SmallDenseMap<TableId, TableId, 8> SoftenedFloats;
107 
108  /// For floating-point nodes that have a smaller precision than the smallest
109  /// supported precision, this map indicates what promoted value to use.
110  SmallDenseMap<TableId, TableId, 8> PromotedFloats;
111 
112  /// For floating-point nodes that have a smaller precision than the smallest
113  /// supported precision, this map indicates the converted value to use.
114  SmallDenseMap<TableId, TableId, 8> SoftPromotedHalfs;
115 
116  /// For float nodes that need to be expanded this map indicates which operands
117  /// are the expanded version of the input.
119 
120  /// For nodes that are <1 x ty>, this map indicates the scalar value of type
121  /// 'ty' to use.
122  SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;
123 
124  /// For nodes that need to be split this map indicates which operands are the
125  /// expanded version of the input.
127 
128  /// For vector nodes that need to be widened, indicates the widened value to
129  /// use.
130  SmallDenseMap<TableId, TableId, 8> WidenedVectors;
131 
132  /// For values that have been replaced with another, indicates the replacement
133  /// value to use.
134  SmallDenseMap<TableId, TableId, 8> ReplacedValues;
135 
136  /// This defines a worklist of nodes to process. In order to be pushed onto
137  /// this worklist, all operands of a node must have already been processed.
138  SmallVector<SDNode*, 128> Worklist;
139 
140  TableId getTableId(SDValue V) {
141  assert(V.getNode() && "Getting TableId on SDValue()");
142 
143  auto I = ValueToIdMap.find(V);
144  if (I != ValueToIdMap.end()) {
145  // replace if there's been a shift.
146  RemapId(I->second);
147  assert(I->second && "All Ids should be nonzero");
148  return I->second;
149  }
150  // Add if it's not there.
151  ValueToIdMap.insert(std::make_pair(V, NextValueId));
152  IdToValueMap.insert(std::make_pair(NextValueId, V));
153  ++NextValueId;
154  assert(NextValueId != 0 &&
155  "Ran out of Ids. Increase id type size or add compactification");
156  return NextValueId - 1;
157  }
158 
159  const SDValue &getSDValue(TableId &Id) {
160  RemapId(Id);
161  assert(Id && "TableId should be non-zero");
162  auto I = IdToValueMap.find(Id);
163  assert(I != IdToValueMap.end() && "cannot find Id in map");
164  return I->second;
165  }
166 
167 public:
169  : TLI(dag.getTargetLoweringInfo()), DAG(dag),
170  ValueTypeActions(TLI.getValueTypeActions()) {
172  "Too many value types for ValueTypeActions to hold!");
173  }
174 
175  /// This is the main entry point for the type legalizer. This does a
176  /// top-down traversal of the dag, legalizing types as it goes. Returns
177  /// "true" if it made any changes.
178  bool run();
179 
180  void NoteDeletion(SDNode *Old, SDNode *New) {
181  assert(Old != New && "node replaced with self");
182  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
183  TableId NewId = getTableId(SDValue(New, i));
184  TableId OldId = getTableId(SDValue(Old, i));
185 
186  if (OldId != NewId) {
187  ReplacedValues[OldId] = NewId;
188 
189  // Delete Node from tables. We cannot do this when OldId == NewId,
190  // because NewId can still have table references to it in
191  // ReplacedValues.
192  IdToValueMap.erase(OldId);
193  PromotedIntegers.erase(OldId);
194  ExpandedIntegers.erase(OldId);
195  SoftenedFloats.erase(OldId);
196  PromotedFloats.erase(OldId);
197  SoftPromotedHalfs.erase(OldId);
198  ExpandedFloats.erase(OldId);
199  ScalarizedVectors.erase(OldId);
200  SplitVectors.erase(OldId);
201  WidenedVectors.erase(OldId);
202  }
203 
204  ValueToIdMap.erase(SDValue(Old, i));
205  }
206  }
207 
208  SelectionDAG &getDAG() const { return DAG; }
209 
210 private:
211  SDNode *AnalyzeNewNode(SDNode *N);
212  void AnalyzeNewValue(SDValue &Val);
213  void PerformExpensiveChecks();
214  void RemapId(TableId &Id);
215  void RemapValue(SDValue &V);
216 
217  // Common routines.
218  SDValue BitConvertToInteger(SDValue Op);
219  SDValue BitConvertVectorToIntegerVector(SDValue Op);
220  SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
221  bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
222  bool CustomWidenLowerNode(SDNode *N, EVT VT);
223 
224  /// Replace each result of the given MERGE_VALUES node with the corresponding
225  /// input operand, except for the result 'ResNo', for which the corresponding
226  /// input operand is returned.
227  SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
228 
229  SDValue JoinIntegers(SDValue Lo, SDValue Hi);
230 
231  std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
232 
233  SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
234 
235  void ReplaceValueWith(SDValue From, SDValue To);
236  void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
237  void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT,
238  SDValue &Lo, SDValue &Hi);
239 
240  //===--------------------------------------------------------------------===//
241  // Integer Promotion Support: LegalizeIntegerTypes.cpp
242  //===--------------------------------------------------------------------===//
243 
244  /// Given a processed operand Op which was promoted to a larger integer type,
245  /// this returns the promoted value. The low bits of the promoted value
246  /// corresponding to the original type are exactly equal to Op.
247  /// The extra bits contain rubbish, so the promoted value may need to be zero-
248  /// or sign-extended from the original type before it is usable (the helpers
249  /// SExtPromotedInteger and ZExtPromotedInteger can do this for you).
250  /// For example, if Op is an i16 and was promoted to an i32, then this method
251  /// returns an i32, the lower 16 bits of which coincide with Op, and the upper
252  /// 16 bits of which contain rubbish.
253  SDValue GetPromotedInteger(SDValue Op) {
254  TableId &PromotedId = PromotedIntegers[getTableId(Op)];
255  SDValue PromotedOp = getSDValue(PromotedId);
256  assert(PromotedOp.getNode() && "Operand wasn't promoted?");
257  return PromotedOp;
258  }
259  void SetPromotedInteger(SDValue Op, SDValue Result);
260 
261  /// Get a promoted operand and sign extend it to the final size.
262  SDValue SExtPromotedInteger(SDValue Op) {
263  EVT OldVT = Op.getValueType();
264  SDLoc dl(Op);
265  Op = GetPromotedInteger(Op);
266  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,
267  DAG.getValueType(OldVT));
268  }
269 
270  /// Get a promoted operand and zero extend it to the final size.
271  SDValue ZExtPromotedInteger(SDValue Op) {
272  EVT OldVT = Op.getValueType();
273  SDLoc dl(Op);
274  Op = GetPromotedInteger(Op);
275  return DAG.getZeroExtendInReg(Op, dl, OldVT);
276  }
277 
278  // Get a promoted operand and sign or zero extend it to the final size
279  // (depending on TargetLoweringInfo::isSExtCheaperThanZExt). For a given
280  // subtarget and type, the choice of sign or zero-extension will be
281  // consistent.
282  SDValue SExtOrZExtPromotedInteger(SDValue Op) {
283  EVT OldVT = Op.getValueType();
284  SDLoc DL(Op);
285  Op = GetPromotedInteger(Op);
286  if (TLI.isSExtCheaperThanZExt(OldVT, Op.getValueType()))
287  return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), Op,
288  DAG.getValueType(OldVT));
289  return DAG.getZeroExtendInReg(Op, DL, OldVT);
290  }
291 
292  // Integer Result Promotion.
293  void PromoteIntegerResult(SDNode *N, unsigned ResNo);
294  SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
295  SDValue PromoteIntRes_AssertSext(SDNode *N);
296  SDValue PromoteIntRes_AssertZext(SDNode *N);
297  SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
298  SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
299  SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
300  SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
301  SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
302  SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
303  SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
304  SDValue PromoteIntRes_SPLAT_VECTOR(SDNode *N);
305  SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
306  SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
307  SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
308  SDValue PromoteIntRes_BITCAST(SDNode *N);
309  SDValue PromoteIntRes_BSWAP(SDNode *N);
310  SDValue PromoteIntRes_BITREVERSE(SDNode *N);
311  SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
312  SDValue PromoteIntRes_Constant(SDNode *N);
313  SDValue PromoteIntRes_CTLZ(SDNode *N);
314  SDValue PromoteIntRes_CTPOP(SDNode *N);
315  SDValue PromoteIntRes_CTTZ(SDNode *N);
316  SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
317  SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
318  SDValue PromoteIntRes_FP_TO_FP16(SDNode *N);
319  SDValue PromoteIntRes_FREEZE(SDNode *N);
320  SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
321  SDValue PromoteIntRes_LOAD(LoadSDNode *N);
322  SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
323  SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
324  SDValue PromoteIntRes_Overflow(SDNode *N);
325  SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
326  SDValue PromoteIntRes_SELECT(SDNode *N);
327  SDValue PromoteIntRes_VSELECT(SDNode *N);
328  SDValue PromoteIntRes_SELECT_CC(SDNode *N);
329  SDValue PromoteIntRes_SETCC(SDNode *N);
330  SDValue PromoteIntRes_SHL(SDNode *N);
331  SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
332  SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
333  SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
334  SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
335  SDValue PromoteIntRes_SRA(SDNode *N);
336  SDValue PromoteIntRes_SRL(SDNode *N);
337  SDValue PromoteIntRes_TRUNCATE(SDNode *N);
338  SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
339  SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
340  SDValue PromoteIntRes_UNDEF(SDNode *N);
341  SDValue PromoteIntRes_VAARG(SDNode *N);
342  SDValue PromoteIntRes_VSCALE(SDNode *N);
343  SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
344  SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
345  SDValue PromoteIntRes_MULFIX(SDNode *N);
346  SDValue PromoteIntRes_DIVFIX(SDNode *N);
347  SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N);
348  SDValue PromoteIntRes_VECREDUCE(SDNode *N);
349  SDValue PromoteIntRes_ABS(SDNode *N);
350 
351  // Integer Operand Promotion.
352  bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
353  SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
354  SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
355  SDValue PromoteIntOp_BITCAST(SDNode *N);
356  SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
357  SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
358  SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);
359  SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);
360  SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
361  SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
362  SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
363  SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
364  SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N);
365  SDValue PromoteIntOp_SPLAT_VECTOR(SDNode *N);
366  SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
367  SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
368  SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
369  SDValue PromoteIntOp_Shift(SDNode *N);
370  SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
371  SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
372  SDValue PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N);
373  SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
374  SDValue PromoteIntOp_TRUNCATE(SDNode *N);
375  SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
376  SDValue PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N);
377  SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
378  SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
379  SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
380  SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
381  SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
382  SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
383  SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
384  SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
385  SDValue PromoteIntOp_FIX(SDNode *N);
386  SDValue PromoteIntOp_FPOWI(SDNode *N);
387  SDValue PromoteIntOp_VECREDUCE(SDNode *N);
388 
389  void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
390 
391  //===--------------------------------------------------------------------===//
392  // Integer Expansion Support: LegalizeIntegerTypes.cpp
393  //===--------------------------------------------------------------------===//
394 
395  /// Given a processed operand Op which was expanded into two integers of half
396  /// the size, this returns the two halves. The low bits of Op are exactly
397  /// equal to the bits of Lo; the high bits exactly equal Hi.
398  /// For example, if Op is an i64 which was expanded into two i32's, then this
399  /// method returns the two i32's, with Lo being equal to the lower 32 bits of
400  /// Op, and Hi being equal to the upper 32 bits.
401  void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
402  void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
403 
404  // Integer Result Expansion.
405  void ExpandIntegerResult(SDNode *N, unsigned ResNo);
406  void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
407  void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
408  void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
409  void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
410  void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
411  void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
412  void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
413  void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
414  void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
415  void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);
416  void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
417  void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
418  void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);
419  void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
420  void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi);
421  void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi);
422  void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi);
423  void ExpandIntRes_LLROUND_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
424 
425  void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);
426  void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
427  void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);
428  void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);
429  void ExpandIntRes_ADDSUBCARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
430  void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);
431  void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);
432  void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);
433  void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
434  void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);
435  void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
436  void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
437  void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);
438 
439  void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);
440 
441  void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
442  void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
443  void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
444  void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
445  void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
446  void ExpandIntRes_DIVFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
447 
448  void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
449  void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi);
450 
451  void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
452  SDValue &Lo, SDValue &Hi);
453  bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
454  bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
455 
456  // Integer Operand Expansion.
457  bool ExpandIntegerOperand(SDNode *N, unsigned OpNo);
458  SDValue ExpandIntOp_BR_CC(SDNode *N);
459  SDValue ExpandIntOp_SELECT_CC(SDNode *N);
460  SDValue ExpandIntOp_SETCC(SDNode *N);
461  SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
462  SDValue ExpandIntOp_Shift(SDNode *N);
463  SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
464  SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
465  SDValue ExpandIntOp_TRUNCATE(SDNode *N);
466  SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
467  SDValue ExpandIntOp_RETURNADDR(SDNode *N);
468  SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
469 
470  void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
471  ISD::CondCode &CCCode, const SDLoc &dl);
472 
473  //===--------------------------------------------------------------------===//
474  // Float to Integer Conversion Support: LegalizeFloatTypes.cpp
475  //===--------------------------------------------------------------------===//
476 
477  /// GetSoftenedFloat - Given a processed operand Op which was converted to an
478  /// integer of the same size, this returns the integer. The integer contains
479  /// exactly the same bits as Op - only the type changed. For example, if Op
480  /// is an f32 which was softened to an i32, then this method returns an i32,
481  /// the bits of which coincide with those of Op
482  SDValue GetSoftenedFloat(SDValue Op) {
483  TableId Id = getTableId(Op);
484  auto Iter = SoftenedFloats.find(Id);
485  if (Iter == SoftenedFloats.end()) {
486  assert(isSimpleLegalType(Op.getValueType()) &&
487  "Operand wasn't converted to integer?");
488  return Op;
489  }
490  SDValue SoftenedOp = getSDValue(Iter->second);
491  assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");
492  return SoftenedOp;
493  }
494  void SetSoftenedFloat(SDValue Op, SDValue Result);
495 
496  // Convert Float Results to Integer.
497  void SoftenFloatResult(SDNode *N, unsigned ResNo);
498  SDValue SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC);
499  SDValue SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC);
500  SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
501  SDValue SoftenFloatRes_BITCAST(SDNode *N);
502  SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
503  SDValue SoftenFloatRes_ConstantFP(SDNode *N);
504  SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);
505  SDValue SoftenFloatRes_FABS(SDNode *N);
506  SDValue SoftenFloatRes_FMINNUM(SDNode *N);
507  SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
508  SDValue SoftenFloatRes_FADD(SDNode *N);
509  SDValue SoftenFloatRes_FCBRT(SDNode *N);
510  SDValue SoftenFloatRes_FCEIL(SDNode *N);
511  SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N);
512  SDValue SoftenFloatRes_FCOS(SDNode *N);
513  SDValue SoftenFloatRes_FDIV(SDNode *N);
514  SDValue SoftenFloatRes_FEXP(SDNode *N);
515  SDValue SoftenFloatRes_FEXP2(SDNode *N);
516  SDValue SoftenFloatRes_FFLOOR(SDNode *N);
517  SDValue SoftenFloatRes_FLOG(SDNode *N);
518  SDValue SoftenFloatRes_FLOG2(SDNode *N);
519  SDValue SoftenFloatRes_FLOG10(SDNode *N);
520  SDValue SoftenFloatRes_FMA(SDNode *N);
521  SDValue SoftenFloatRes_FMUL(SDNode *N);
522  SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);
523  SDValue SoftenFloatRes_FNEG(SDNode *N);
524  SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);
525  SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);
526  SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
527  SDValue SoftenFloatRes_FPOW(SDNode *N);
528  SDValue SoftenFloatRes_FPOWI(SDNode *N);
529  SDValue SoftenFloatRes_FREEZE(SDNode *N);
530  SDValue SoftenFloatRes_FREM(SDNode *N);
531  SDValue SoftenFloatRes_FRINT(SDNode *N);
532  SDValue SoftenFloatRes_FROUND(SDNode *N);
533  SDValue SoftenFloatRes_FROUNDEVEN(SDNode *N);
534  SDValue SoftenFloatRes_FSIN(SDNode *N);
535  SDValue SoftenFloatRes_FSQRT(SDNode *N);
536  SDValue SoftenFloatRes_FSUB(SDNode *N);
537  SDValue SoftenFloatRes_FTRUNC(SDNode *N);
538  SDValue SoftenFloatRes_LOAD(SDNode *N);
539  SDValue SoftenFloatRes_SELECT(SDNode *N);
540  SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
541  SDValue SoftenFloatRes_UNDEF(SDNode *N);
542  SDValue SoftenFloatRes_VAARG(SDNode *N);
543  SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
544 
545  // Convert Float Operand to Integer.
546  bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
547  SDValue SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC);
548  SDValue SoftenFloatOp_BITCAST(SDNode *N);
549  SDValue SoftenFloatOp_BR_CC(SDNode *N);
550  SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
551  SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
552  SDValue SoftenFloatOp_LROUND(SDNode *N);
553  SDValue SoftenFloatOp_LLROUND(SDNode *N);
554  SDValue SoftenFloatOp_LRINT(SDNode *N);
555  SDValue SoftenFloatOp_LLRINT(SDNode *N);
556  SDValue SoftenFloatOp_SELECT_CC(SDNode *N);
557  SDValue SoftenFloatOp_SETCC(SDNode *N);
558  SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
559  SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);
560 
561  //===--------------------------------------------------------------------===//
562  // Float Expansion Support: LegalizeFloatTypes.cpp
563  //===--------------------------------------------------------------------===//
564 
565  /// Given a processed operand Op which was expanded into two floating-point
566  /// values of half the size, this returns the two halves.
567  /// The low bits of Op are exactly equal to the bits of Lo; the high bits
568  /// exactly equal Hi. For example, if Op is a ppcf128 which was expanded
569  /// into two f64's, then this method returns the two f64's, with Lo being
570  /// equal to the lower 64 bits of Op, and Hi to the upper 64 bits.
571  void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
572  void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
573 
574  // Float Result Expansion.
575  void ExpandFloatResult(SDNode *N, unsigned ResNo);
576  void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
577  void ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
578  SDValue &Lo, SDValue &Hi);
579  void ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
580  SDValue &Lo, SDValue &Hi);
581  void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
582  void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
583  void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
584  void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
585  void ExpandFloatRes_FCBRT (SDNode *N, SDValue &Lo, SDValue &Hi);
586  void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
587  void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
588  void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);
589  void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
590  void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
591  void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);
592  void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);
593  void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);
594  void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);
595  void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);
596  void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);
597  void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);
598  void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
599  void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);
600  void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
601  void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);
602  void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);
603  void ExpandFloatRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
604  void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);
605  void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
606  void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
607  void ExpandFloatRes_FROUNDEVEN(SDNode *N, SDValue &Lo, SDValue &Hi);
608  void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);
609  void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);
610  void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
611  void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);
612  void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
613  void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
614 
615  // Float Operand Expansion.
616  bool ExpandFloatOperand(SDNode *N, unsigned OpNo);
617  SDValue ExpandFloatOp_BR_CC(SDNode *N);
618  SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);
619  SDValue ExpandFloatOp_FP_ROUND(SDNode *N);
620  SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N);
621  SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N);
622  SDValue ExpandFloatOp_LROUND(SDNode *N);
623  SDValue ExpandFloatOp_LLROUND(SDNode *N);
624  SDValue ExpandFloatOp_LRINT(SDNode *N);
625  SDValue ExpandFloatOp_LLRINT(SDNode *N);
626  SDValue ExpandFloatOp_SELECT_CC(SDNode *N);
627  SDValue ExpandFloatOp_SETCC(SDNode *N);
628  SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);
629 
630  void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
631  ISD::CondCode &CCCode, const SDLoc &dl);
632 
633  //===--------------------------------------------------------------------===//
634  // Float promotion support: LegalizeFloatTypes.cpp
635  //===--------------------------------------------------------------------===//
636 
637  SDValue GetPromotedFloat(SDValue Op) {
638  TableId &PromotedId = PromotedFloats[getTableId(Op)];
639  SDValue PromotedOp = getSDValue(PromotedId);
640  assert(PromotedOp.getNode() && "Operand wasn't promoted?");
641  return PromotedOp;
642  }
643  void SetPromotedFloat(SDValue Op, SDValue Result);
644 
645  void PromoteFloatResult(SDNode *N, unsigned ResNo);
646  SDValue PromoteFloatRes_BITCAST(SDNode *N);
647  SDValue PromoteFloatRes_BinOp(SDNode *N);
648  SDValue PromoteFloatRes_ConstantFP(SDNode *N);
649  SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
650  SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
651  SDValue PromoteFloatRes_FMAD(SDNode *N);
652  SDValue PromoteFloatRes_FPOWI(SDNode *N);
653  SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
654  SDValue PromoteFloatRes_LOAD(SDNode *N);
655  SDValue PromoteFloatRes_SELECT(SDNode *N);
656  SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
657  SDValue PromoteFloatRes_UnaryOp(SDNode *N);
658  SDValue PromoteFloatRes_UNDEF(SDNode *N);
659  SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
660  SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
661 
662  bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
663  SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
664  SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
665  SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
666  SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
667  SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
668  SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
669  SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
670 
671  //===--------------------------------------------------------------------===//
672  // Half soft promotion support: LegalizeFloatTypes.cpp
673  //===--------------------------------------------------------------------===//
674 
675  SDValue GetSoftPromotedHalf(SDValue Op) {
676  TableId &PromotedId = SoftPromotedHalfs[getTableId(Op)];
677  SDValue PromotedOp = getSDValue(PromotedId);
678  assert(PromotedOp.getNode() && "Operand wasn't promoted?");
679  return PromotedOp;
680  }
681  void SetSoftPromotedHalf(SDValue Op, SDValue Result);
682 
683  void SoftPromoteHalfResult(SDNode *N, unsigned ResNo);
684  SDValue SoftPromoteHalfRes_BinOp(SDNode *N);
685  SDValue SoftPromoteHalfRes_BITCAST(SDNode *N);
686  SDValue SoftPromoteHalfRes_ConstantFP(SDNode *N);
687  SDValue SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N);
688  SDValue SoftPromoteHalfRes_FCOPYSIGN(SDNode *N);
689  SDValue SoftPromoteHalfRes_FMAD(SDNode *N);
690  SDValue SoftPromoteHalfRes_FPOWI(SDNode *N);
691  SDValue SoftPromoteHalfRes_FP_ROUND(SDNode *N);
692  SDValue SoftPromoteHalfRes_LOAD(SDNode *N);
693  SDValue SoftPromoteHalfRes_SELECT(SDNode *N);
694  SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);
695  SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);
696  SDValue SoftPromoteHalfRes_XINT_TO_FP(SDNode *N);
697  SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);
698 
699  bool SoftPromoteHalfOperand(SDNode *N, unsigned OpNo);
700  SDValue SoftPromoteHalfOp_BITCAST(SDNode *N);
701  SDValue SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
702  SDValue SoftPromoteHalfOp_FP_EXTEND(SDNode *N);
703  SDValue SoftPromoteHalfOp_FP_TO_XINT(SDNode *N);
704  SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
705  SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
706  SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
707 
708  //===--------------------------------------------------------------------===//
709  // Scalarization Support: LegalizeVectorTypes.cpp
710  //===--------------------------------------------------------------------===//
711 
712  /// Given a processed one-element vector Op which was scalarized to its
713  /// element type, this returns the element. For example, if Op is a v1i32,
714  /// Op = < i32 val >, this method returns val, an i32.
715  SDValue GetScalarizedVector(SDValue Op) {
716  TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];
717  SDValue ScalarizedOp = getSDValue(ScalarizedId);
718  assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");
719  return ScalarizedOp;
720  }
721  void SetScalarizedVector(SDValue Op, SDValue Result);
722 
723  // Vector Result Scalarization: <1 x ty> -> ty.
724  void ScalarizeVectorResult(SDNode *N, unsigned ResNo);
725  SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
726  SDValue ScalarizeVecRes_BinOp(SDNode *N);
727  SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
728  SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
729  SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
730  SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
731  SDValue ScalarizeVecRes_InregOp(SDNode *N);
732  SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
733 
734  SDValue ScalarizeVecRes_BITCAST(SDNode *N);
735  SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
736  SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
737  SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
738  SDValue ScalarizeVecRes_FPOWI(SDNode *N);
739  SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
740  SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
741  SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
742  SDValue ScalarizeVecRes_VSELECT(SDNode *N);
743  SDValue ScalarizeVecRes_SELECT(SDNode *N);
744  SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
745  SDValue ScalarizeVecRes_SETCC(SDNode *N);
746  SDValue ScalarizeVecRes_UNDEF(SDNode *N);
747  SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
748 
749  SDValue ScalarizeVecRes_FIX(SDNode *N);
750 
751  // Vector Operand Scalarization: <1 x ty> -> ty.
752  bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
753  SDValue ScalarizeVecOp_BITCAST(SDNode *N);
754  SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
755  SDValue ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N);
756  SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
757  SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
758  SDValue ScalarizeVecOp_VSELECT(SDNode *N);
759  SDValue ScalarizeVecOp_VSETCC(SDNode *N);
760  SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
761  SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
762  SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);
763  SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
764 
765  //===--------------------------------------------------------------------===//
766  // Vector Splitting Support: LegalizeVectorTypes.cpp
767  //===--------------------------------------------------------------------===//
768 
769  /// Given a processed vector Op which was split into vectors of half the size,
770  /// this method returns the halves. The first elements of Op coincide with the
771  /// elements of Lo; the remaining elements of Op coincide with the elements of
772  /// Hi: Op is what you would get by concatenating Lo and Hi.
773  /// For example, if Op is a v8i32 that was split into two v4i32's, then this
774  /// method returns the two v4i32's, with Lo corresponding to the first 4
775  /// elements of Op, and Hi to the last 4 elements.
776  void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
777  void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
778 
779  // Helper function for incrementing the pointer when splitting
780  // memory operations
781  void IncrementPointer(MemSDNode *N, EVT MemVT,
782  MachinePointerInfo &MPI, SDValue &Ptr);
783 
784  // Vector Result Splitting: <128 x ty> -> 2 x <64 x ty>.
785  void SplitVectorResult(SDNode *N, unsigned ResNo);
786  void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
787  void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
788  void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
789  void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
790  void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
791  void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
792  void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
793  void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
794  SDValue &Lo, SDValue &Hi);
795 
796  void SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi);
797 
798  void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
799  void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
800  void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
801  void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
802  void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
803  void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi);
804  void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi);
805  void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
806  void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
807  void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
808  void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
809  void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
810  void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
811  void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
812  SDValue &Hi);
813  void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);
814 
815  // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
816  bool SplitVectorOperand(SDNode *N, unsigned OpNo);
817  SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
818  SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
819  SDValue SplitVecOp_UnaryOp(SDNode *N);
820  SDValue SplitVecOp_TruncateHelper(SDNode *N);
821 
822  SDValue SplitVecOp_BITCAST(SDNode *N);
823  SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
824  SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
825  SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
826  SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
827  SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
828  SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
829  SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, unsigned OpNo);
830  SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
831  SDValue SplitVecOp_VSETCC(SDNode *N);
832  SDValue SplitVecOp_FP_ROUND(SDNode *N);
833  SDValue SplitVecOp_FCOPYSIGN(SDNode *N);
834 
835  //===--------------------------------------------------------------------===//
836  // Vector Widening Support: LegalizeVectorTypes.cpp
837  //===--------------------------------------------------------------------===//
838 
839  /// Given a processed vector Op which was widened into a larger vector, this
840  /// method returns the larger vector. The elements of the returned vector
841  /// consist of the elements of Op followed by elements containing rubbish.
842  /// For example, if Op is a v2i32 that was widened to a v4i32, then this
843  /// method returns a v4i32 for which the first two elements are the same as
844  /// those of Op, while the last two elements contain rubbish.
845  SDValue GetWidenedVector(SDValue Op) {
846  TableId &WidenedId = WidenedVectors[getTableId(Op)];
847  SDValue WidenedOp = getSDValue(WidenedId);
848  assert(WidenedOp.getNode() && "Operand wasn't widened?");
849  return WidenedOp;
850  }
851  void SetWidenedVector(SDValue Op, SDValue Result);
852 
853  // Widen Vector Result Promotion.
854  void WidenVectorResult(SDNode *N, unsigned ResNo);
855  SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
856  SDValue WidenVecRes_BITCAST(SDNode* N);
857  SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
858  SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
859  SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
860  SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
861  SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
862  SDValue WidenVecRes_LOAD(SDNode* N);
863  SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
864  SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
865  SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
866  SDValue WidenVecRes_SELECT(SDNode* N);
867  SDValue WidenVSELECTAndMask(SDNode *N);
868  SDValue WidenVecRes_SELECT_CC(SDNode* N);
869  SDValue WidenVecRes_SETCC(SDNode* N);
870  SDValue WidenVecRes_STRICT_FSETCC(SDNode* N);
871  SDValue WidenVecRes_UNDEF(SDNode *N);
872  SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
873 
874  SDValue WidenVecRes_Ternary(SDNode *N);
875  SDValue WidenVecRes_Binary(SDNode *N);
876  SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
877  SDValue WidenVecRes_BinaryWithExtraScalarOp(SDNode *N);
878  SDValue WidenVecRes_StrictFP(SDNode *N);
879  SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);
880  SDValue WidenVecRes_Convert(SDNode *N);
881  SDValue WidenVecRes_Convert_StrictFP(SDNode *N);
882  SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
883  SDValue WidenVecRes_POWI(SDNode *N);
884  SDValue WidenVecRes_Shift(SDNode *N);
885  SDValue WidenVecRes_Unary(SDNode *N);
886  SDValue WidenVecRes_InregOp(SDNode *N);
887 
888  // Widen Vector Operand.
889  bool WidenVectorOperand(SDNode *N, unsigned OpNo);
890  SDValue WidenVecOp_BITCAST(SDNode *N);
891  SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
892  SDValue WidenVecOp_EXTEND(SDNode *N);
893  SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
894  SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
895  SDValue WidenVecOp_STORE(SDNode* N);
896  SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
897  SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
898  SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
899  SDValue WidenVecOp_SETCC(SDNode* N);
900  SDValue WidenVecOp_STRICT_FSETCC(SDNode* N);
901  SDValue WidenVecOp_VSELECT(SDNode *N);
902 
903  SDValue WidenVecOp_Convert(SDNode *N);
904  SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
905  SDValue WidenVecOp_VECREDUCE(SDNode *N);
906 
907  /// Helper function to generate a set of operations to perform
908  /// a vector operation for a wider type.
909  ///
910  SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE);
911 
912  //===--------------------------------------------------------------------===//
913  // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
914  //===--------------------------------------------------------------------===//
915 
916  /// Helper function to generate a set of loads to load a vector with a
917  /// resulting wider type. It takes:
918  /// LdChain: list of chains for the load to be generated.
919  /// Ld: load to widen
920  SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
921  LoadSDNode *LD);
922 
923  /// Helper function to generate a set of extension loads to load a vector with
924  /// a resulting wider type. It takes:
925  /// LdChain: list of chains for the load to be generated.
926  /// Ld: load to widen
927  /// ExtType: extension element type
928  SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
930 
931  /// Helper function to generate a set of stores to store a widen vector into
932  /// non-widen memory.
933  /// StChain: list of chains for the stores we have generated
934  /// ST: store of a widen value
935  void GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST);
936 
937  /// Helper function to generate a set of stores to store a truncate widen
938  /// vector into non-widen memory.
939  /// StChain: list of chains for the stores we have generated
940  /// ST: store of a widen value
941  void GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
942  StoreSDNode *ST);
943 
944  /// Modifies a vector input (widen or narrows) to a vector of NVT. The
945  /// input vector must have the same element type as NVT.
946  /// When FillWithZeroes is "on" the vector will be widened with zeroes.
947  /// By default, the vector will be widened with undefined values.
948  SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
949 
950  /// Return a mask of vector type MaskVT to replace InMask. Also adjust
951  /// MaskVT to ToMaskVT if needed with vector extension or truncation.
952  SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
953 
954  //===--------------------------------------------------------------------===//
955  // Generic Splitting: LegalizeTypesGeneric.cpp
956  //===--------------------------------------------------------------------===//
957 
958  // Legalization methods which only use that the illegal type is split into two
959  // not necessarily identical types. As such they can be used for splitting
960  // vectors and expanding integers and floats.
961 
962  void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
963  if (Op.getValueType().isVector())
964  GetSplitVector(Op, Lo, Hi);
965  else if (Op.getValueType().isInteger())
966  GetExpandedInteger(Op, Lo, Hi);
967  else
968  GetExpandedFloat(Op, Lo, Hi);
969  }
970 
971  /// Use ISD::EXTRACT_ELEMENT nodes to extract the low and high parts of the
972  /// given value.
973  void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
974 
975  // Generic Result Splitting.
976  void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
977  SDValue &Lo, SDValue &Hi);
978  void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi);
979  void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
980  void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
981  void SplitRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
982 
983  void SplitVSETCC(const SDNode *N);
984 
985  //===--------------------------------------------------------------------===//
986  // Generic Expansion: LegalizeTypesGeneric.cpp
987  //===--------------------------------------------------------------------===//
988 
989  // Legalization methods which only use that the illegal type is split into two
990  // identical types of half the size, and that the Lo/Hi part is stored first
991  // in memory on little/big-endian machines, followed by the Hi/Lo part. As
992  // such they can be used for expanding integers and floats.
993 
994  void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
995  if (Op.getValueType().isInteger())
996  GetExpandedInteger(Op, Lo, Hi);
997  else
998  GetExpandedFloat(Op, Lo, Hi);
999  }
1000 
1001 
1002  /// This function will split the integer \p Op into \p NumElements
1003  /// operations of type \p EltVT and store them in \p Ops.
1004  void IntegerToVector(SDValue Op, unsigned NumElements,
1005  SmallVectorImpl<SDValue> &Ops, EVT EltVT);
1006 
1007  // Generic Result Expansion.
1008  void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo,
1009  SDValue &Lo, SDValue &Hi);
1010  void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);
1011  void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);
1012  void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);
1013  void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
1014  void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);
1015  void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);
1016 
1017  // Generic Operand Expansion.
1018  SDValue ExpandOp_BITCAST (SDNode *N);
1019  SDValue ExpandOp_BUILD_VECTOR (SDNode *N);
1020  SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N);
1021  SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);
1022  SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);
1023  SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo);
1024 };
1025 
1026 } // end namespace llvm.
1027 
1028 #endif
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:146
This takes an arbitrary SelectionDAG as input and hacks on it until only value types the target machi...
Definition: LegalizeTypes.h:31
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:424
This is an SDNode representing atomic operations.
SelectionDAG & getDAG() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This class is used to represent an MSTORE node.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:1204
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:142
This class is used to represent ISD::STORE nodes.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1184
constexpr double e
Definition: MathExtras.h:58
#define LLVM_LIBRARY_VISIBILITY
LLVM_LIBRARY_VISIBILITY - If a class marked with this attribute is linked into a shared library...
Definition: Compiler.h:130
Extended Value Type.
Definition: ValueTypes.h:35
NodeIdFlags
This pass uses the NodeId on the SDNodes to hold information about the state of the node...
Definition: LegalizeTypes.h:37
This class contains a discriminated union of information about pointers in memory operands...
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
void NoteDeletion(SDNode *Old, SDNode *New)
BlockVerifier::State From
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:223
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:883
This is an abstract virtual class for memory operations.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Class for arbitrary precision integers.
Definition: APInt.h:69
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:156
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:665
#define I(x, y, z)
Definition: MD5.cpp:59
#define N
This class is used to represent an MSCATTER node.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
This class is used to represent an MLOAD node.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:131
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
LLVMContext * getContext() const
Definition: SelectionDAG.h:431
This file describes how to lower LLVM code to machine code.
DAGTypeLegalizer(SelectionDAG &dag)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This class is used to represent ISD::LOAD nodes.