/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Bug Summary

File:	llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
Warning:	line 2294, column 62 The result of the right shift is undefined due to shifting by '32', which is greater or equal to the width of type 'unsigned int'

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AMDGPULegalizerInfo.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fdenormal-fp-math=ieee,ieee -fdenormal-fp-math-f32=ieee,ieee -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-11/lib/clang/11.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/build-llvm/include -I /build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-11/lib/clang/11.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/build-llvm/lib/Target/AMDGPU -fdebug-prefix-map=/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62=. -ferror-limit 19 -fmessage-length 0 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-03-05-084736-29294-1 -x c++ /build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

→

1//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//

14#if defined(_MSC_VER) || defined(__MINGW32__)
15// According to Microsoft, one must set _USE_MATH_DEFINES in order to get M_PI
16// from the Visual C++ cmath / math.h headers:
17// https://docs.microsoft.com/en-us/cpp/c-runtime-library/math-constants?view=vs-2019
18#define _USE_MATH_DEFINES
19#endif

21#include "AMDGPULegalizerInfo.h"

23#include "AMDGPU.h"
24#include "AMDGPUGlobalISelUtils.h"
25#include "AMDGPUTargetMachine.h"
26#include "SIMachineFunctionInfo.h"
27#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
28#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
30#include "llvm/CodeGen/TargetOpcodes.h"
31#include "llvm/CodeGen/ValueTypes.h"
32#include "llvm/IR/DerivedTypes.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/Type.h"
35#include "llvm/Support/Debug.h"

37#define DEBUG_TYPE"amdgpu-legalinfo" "amdgpu-legalinfo"

39using namespace llvm;
40using namespace LegalizeActions;
41using namespace LegalizeMutations;
42using namespace LegalityPredicates;
43using namespace MIPatternMatch;

45// Round the number of elements to the next power of two elements
46static LLT getPow2VectorType(LLT Ty) {
unsigned NElts = Ty.getNumElements();
unsigned Pow2NElts = 1 <<  Log2_32_Ceil(NElts);
return Ty.changeNumElements(Pow2NElts);
50}

52// Round the number of bits to the next power of two bits
53static LLT getPow2ScalarType(LLT Ty) {
unsigned Bits = Ty.getSizeInBits();
unsigned Pow2Bits = 1 <<  Log2_32_Ceil(Bits);
return LLT::scalar(Pow2Bits);
57}

59static LegalityPredicate isMultiple32(unsigned TypeIdx,
                                    unsigned MaxSize = 1024) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  const LLT EltTy = Ty.getScalarType();
  return Ty.getSizeInBits() <= MaxSize && EltTy.getSizeInBits() % 32 == 0;
};
66}

68static LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size) {
return [=](const LegalityQuery &Query) {
  return Query.Types[TypeIdx].getSizeInBits() == Size;
};
72}

74static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  return Ty.isVector() &&
         Ty.getNumElements() % 2 != 0 &&
         Ty.getElementType().getSizeInBits() < 32 &&
         Ty.getSizeInBits() % 32 != 0;
};
82}

84static LegalityPredicate isWideVec16(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  const LLT EltTy = Ty.getScalarType();
  return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
};
90}

92static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  const LLT EltTy = Ty.getElementType();
  return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
};
98}

100static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  const LLT EltTy = Ty.getElementType();
  unsigned Size = Ty.getSizeInBits();
  unsigned Pieces = (Size + 63) / 64;
  unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
  return std::make_pair(TypeIdx, LLT::scalarOrVector(NewNumElts, EltTy));
};
109}

111// Increase the number of vector elements to reach the next multiple of 32-bit
112// type.
113static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];

  const LLT EltTy = Ty.getElementType();
  const int Size = Ty.getSizeInBits();
  const int EltSize = EltTy.getSizeInBits();
  const int NextMul32 = (Size + 31) / 32;

  assert(EltSize < 32)((EltSize < 32) ? static_cast<void> (0) : __assert_fail
 ("EltSize < 32", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 122, __PRETTY_FUNCTION__));

  const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
  return std::make_pair(TypeIdx, LLT::vector(NewNumElts, EltTy));
};
127}

129static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
return [=](const LegalityQuery &Query) {
  const LLT QueryTy = Query.Types[TypeIdx];
  return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
};
134}

136static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
return [=](const LegalityQuery &Query) {
  const LLT QueryTy = Query.Types[TypeIdx];
  return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
};
141}

143static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT QueryTy = Query.Types[TypeIdx];
  return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
};
148}

150// Any combination of 32 or 64-bit elements up to 1024 bits, and multiples of
151// v2s16.
152static LegalityPredicate isRegisterType(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  if (Ty.isVector()) {
    const int EltSize = Ty.getElementType().getSizeInBits();
    return EltSize == 32 || EltSize == 64 ||
          (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
           EltSize == 128 || EltSize == 256;
  }

  return Ty.getSizeInBits() % 32 == 0 && Ty.getSizeInBits() <= 1024;
};
164}

166static LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT Type) {
return [=](const LegalityQuery &Query) {
  const LLT QueryTy = Query.Types[TypeIdx];
  return QueryTy.isVector() && QueryTy.getElementType() == Type;
};
171}

173static LegalityPredicate isWideScalarTruncStore(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
  const LLT Ty = Query.Types[TypeIdx];
  return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
         Query.MMODescrs[0].SizeInBits < Ty.getSizeInBits();
};
179}

181static LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1) {
return [=](const LegalityQuery &Query) {
  return Query.Types[TypeIdx0].getSizeInBits() <
         Query.Types[TypeIdx1].getSizeInBits();
};
186}

188static LegalityPredicate greaterThan(unsigned TypeIdx0, unsigned TypeIdx1) {
return [=](const LegalityQuery &Query) {
  return Query.Types[TypeIdx0].getSizeInBits() >
         Query.Types[TypeIdx1].getSizeInBits();
};
193}

195AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
                                       const GCNTargetMachine &TM)
:  ST(ST_) {
using namespace TargetOpcode;

auto GetAddrSpacePtr = [&TM](unsigned AS) {
  return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
};

const LLT S1 = LLT::scalar(1);
const LLT S16 = LLT::scalar(16);
const LLT S32 = LLT::scalar(32);
const LLT S64 = LLT::scalar(64);
const LLT S128 = LLT::scalar(128);
const LLT S256 = LLT::scalar(256);
const LLT S1024 = LLT::scalar(1024);

const LLT V2S16 = LLT::vector(2, 16);
const LLT V4S16 = LLT::vector(4, 16);

const LLT V2S32 = LLT::vector(2, 32);
const LLT V3S32 = LLT::vector(3, 32);
const LLT V4S32 = LLT::vector(4, 32);
const LLT V5S32 = LLT::vector(5, 32);
const LLT V6S32 = LLT::vector(6, 32);
const LLT V7S32 = LLT::vector(7, 32);
const LLT V8S32 = LLT::vector(8, 32);
const LLT V9S32 = LLT::vector(9, 32);
const LLT V10S32 = LLT::vector(10, 32);
const LLT V11S32 = LLT::vector(11, 32);
const LLT V12S32 = LLT::vector(12, 32);
const LLT V13S32 = LLT::vector(13, 32);
const LLT V14S32 = LLT::vector(14, 32);
const LLT V15S32 = LLT::vector(15, 32);
const LLT V16S32 = LLT::vector(16, 32);
const LLT V32S32 = LLT::vector(32, 32);

const LLT V2S64 = LLT::vector(2, 64);
const LLT V3S64 = LLT::vector(3, 64);
const LLT V4S64 = LLT::vector(4, 64);
const LLT V5S64 = LLT::vector(5, 64);
const LLT V6S64 = LLT::vector(6, 64);
const LLT V7S64 = LLT::vector(7, 64);
const LLT V8S64 = LLT::vector(8, 64);
const LLT V16S64 = LLT::vector(16, 64);

std::initializer_list<LLT> AllS32Vectors =
  {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
   V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32, V32S32};
std::initializer_list<LLT> AllS64Vectors =
  {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};

const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT);
const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);

const LLT CodePtr = FlatPtr;

const std::initializer_list<LLT> AddrSpaces64 = {
  GlobalPtr, ConstantPtr, FlatPtr
};

const std::initializer_list<LLT> AddrSpaces32 = {
  LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
};

const std::initializer_list<LLT> FPTypesBase = {
  S32, S64
};

const std::initializer_list<LLT> FPTypes16 = {
  S32, S64, S16
};

const std::initializer_list<LLT> FPTypesPK16 = {
  S32, S64, S16, V2S16
};

const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;

setAction({G_BRCOND, S1}, Legal); // VCC branches
setAction({G_BRCOND, S32}, Legal); // SCC branches

// TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
// elements for v3s16
getActionDefinitionsBuilder(G_PHI)
  .legalFor({S32, S64, V2S16, V4S16, S1, S128, S256})
  .legalFor(AllS32Vectors)
  .legalFor(AllS64Vectors)
  .legalFor(AddrSpaces64)
  .legalFor(AddrSpaces32)
  .clampScalar(0, S32, S256)
  .widenScalarToNextPow2(0, 32)
  .clampMaxNumElements(0, S32, 16)
  .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
  .legalIf(isPointer(0));

if (ST.hasVOP3PInsts()) {
  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
    .legalFor({S32, S16, V2S16})
    .clampScalar(0, S16, S32)
    .clampMaxNumElements(0, S16, 2)
    .scalarize(0)
    .widenScalarToNextPow2(0, 32);
} else if (ST.has16BitInsts()) {
  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
    .legalFor({S32, S16})
    .clampScalar(0, S16, S32)
    .scalarize(0)
    .widenScalarToNextPow2(0, 32);
} else {
  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL})
    .legalFor({S32})
    .clampScalar(0, S32, S32)
    .scalarize(0);
}

// FIXME: Not really legal. Placeholder for custom lowering.
getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_SREM, G_UREM})
  .customFor({S32, S64})
  .clampScalar(0, S32, S64)
  .widenScalarToNextPow2(0, 32)
  .scalarize(0);

getActionDefinitionsBuilder({G_UMULH, G_SMULH})
  .legalFor({S32})
  .clampScalar(0, S32, S32)
  .scalarize(0);

// Report legal for any types we can handle anywhere. For the cases only legal
// on the SALU, RegBankSelect will be able to re-legalize.
getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
  .legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16})
  .clampScalar(0, S32, S64)
  .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
  .fewerElementsIf(vectorWiderThan(0, 64), fewerEltsToSize64Vector(0))
  .widenScalarToNextPow2(0)
  .scalarize(0);

getActionDefinitionsBuilder({G_UADDO, G_USUBO,
                             G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
  .legalFor({{S32, S1}, {S32, S32}})
  .minScalar(0, S32)
  // TODO: .scalarize(0)
  .lower();

getActionDefinitionsBuilder(G_BITCAST)
  // Don't worry about the size constraint.
  .legalIf(all(isRegisterType(0), isRegisterType(1)))
  .lower();


getActionDefinitionsBuilder(G_CONSTANT)
  .legalFor({S1, S32, S64, S16, GlobalPtr,
             LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
  .clampScalar(0, S32, S64)
  .widenScalarToNextPow2(0)
  .legalIf(isPointer(0));

getActionDefinitionsBuilder(G_FCONSTANT)
  .legalFor({S32, S64, S16})
  .clampScalar(0, S16, S64);

getActionDefinitionsBuilder(G_IMPLICIT_DEF)
  .legalFor({S1, S32, S64, S16, V2S32, V4S32, V2S16, V4S16, GlobalPtr,
             ConstantPtr, LocalPtr, FlatPtr, PrivatePtr})
  .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
  .clampScalarOrElt(0, S32, S1024)
  .legalIf(isMultiple32(0))
  .widenScalarToNextPow2(0, 32)
  .clampMaxNumElements(0, S32, 16);

setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
getActionDefinitionsBuilder(G_GLOBAL_VALUE)
  .unsupportedFor({PrivatePtr})
  .custom();
setAction({G_BLOCK_ADDR, CodePtr}, Legal);

auto &FPOpActions = getActionDefinitionsBuilder(
  { G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE})
  .legalFor({S32, S64});
auto &TrigActions = getActionDefinitionsBuilder({G_FSIN, G_FCOS})
  .customFor({S32, S64});
auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV)
  .customFor({S32, S64});

if (ST.has16BitInsts()) {
  if (ST.hasVOP3PInsts())
    FPOpActions.legalFor({S16, V2S16});
  else
    FPOpActions.legalFor({S16});

  TrigActions.customFor({S16});
  FDIVActions.customFor({S16});
}

auto &MinNumMaxNum = getActionDefinitionsBuilder({
    G_FMINNUM, G_FMAXNUM, G_FMINNUM_IEEE, G_FMAXNUM_IEEE});

if (ST.hasVOP3PInsts()) {
  MinNumMaxNum.customFor(FPTypesPK16)
    .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
    .clampMaxNumElements(0, S16, 2)
    .clampScalar(0, S16, S64)
    .scalarize(0);
} else if (ST.has16BitInsts()) {
  MinNumMaxNum.customFor(FPTypes16)
    .clampScalar(0, S16, S64)
    .scalarize(0);
} else {
  MinNumMaxNum.customFor(FPTypesBase)
    .clampScalar(0, S32, S64)
    .scalarize(0);
}

if (ST.hasVOP3PInsts())
  FPOpActions.clampMaxNumElements(0, S16, 2);

FPOpActions
  .scalarize(0)
  .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);

TrigActions
  .scalarize(0)
  .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);

FDIVActions
  .scalarize(0)
  .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);

getActionDefinitionsBuilder({G_FNEG, G_FABS})
  .legalFor(FPTypesPK16)
  .clampMaxNumElements(0, S16, 2)
  .scalarize(0)
  .clampScalar(0, S16, S64);

if (ST.has16BitInsts()) {
  getActionDefinitionsBuilder({G_FSQRT, G_FFLOOR})
    .legalFor({S32, S64, S16})
    .scalarize(0)
    .clampScalar(0, S16, S64);
} else {
  getActionDefinitionsBuilder(G_FSQRT)
    .legalFor({S32, S64})
    .scalarize(0)
    .clampScalar(0, S32, S64);

  if (ST.hasFractBug()) {
    getActionDefinitionsBuilder(G_FFLOOR)
      .customFor({S64})
      .legalFor({S32, S64})
      .scalarize(0)
      .clampScalar(0, S32, S64);
  } else {
    getActionDefinitionsBuilder(G_FFLOOR)
      .legalFor({S32, S64})
      .scalarize(0)
      .clampScalar(0, S32, S64);
  }
}

getActionDefinitionsBuilder(G_FPTRUNC)
  .legalFor({{S32, S64}, {S16, S32}})
  .scalarize(0)
  .lower();

getActionDefinitionsBuilder(G_FPEXT)
  .legalFor({{S64, S32}, {S32, S16}})
  .lowerFor({{S64, S16}}) // FIXME: Implement
  .scalarize(0);

getActionDefinitionsBuilder(G_FSUB)
    // Use actual fsub instruction
    .legalFor({S32})
    // Must use fadd + fneg
    .lowerFor({S64, S16, V2S16})
    .scalarize(0)
    .clampScalar(0, S32, S64);

// Whether this is legal depends on the floating point mode for the function.
auto &FMad = getActionDefinitionsBuilder(G_FMAD);
if (ST.hasMadF16())
  FMad.customFor({S32, S16});
else
  FMad.customFor({S32});
FMad.scalarize(0)
    .lower();

getActionDefinitionsBuilder(G_TRUNC)
  .alwaysLegal();

getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
  .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
             {S32, S1}, {S64, S1}, {S16, S1}})
  .scalarize(0)
  .clampScalar(0, S32, S64)
  .widenScalarToNextPow2(1, 32);

// TODO: Split s1->s64 during regbankselect for VALU.
auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
  .legalFor({{S32, S32}, {S64, S32}, {S16, S32}})
  .lowerFor({{S32, S64}})
  .lowerIf(typeIs(1, S1))
  .customFor({{S64, S64}});
if (ST.has16BitInsts())
  IToFP.legalFor({{S16, S16}});
IToFP.clampScalar(1, S32, S64)
     .scalarize(0)
     .widenScalarToNextPow2(1);

auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
  .legalFor({{S32, S32}, {S32, S64}, {S32, S16}})
  .customFor({{S64, S64}});
if (ST.has16BitInsts())
  FPToI.legalFor({{S16, S16}});
else
  FPToI.minScalar(1, S32);

FPToI.minScalar(0, S32)
     .scalarize(0)
     .lower();

getActionDefinitionsBuilder(G_INTRINSIC_ROUND)
  .scalarize(0)
  .lower();

if (ST.has16BitInsts()) {
  getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
    .legalFor({S16, S32, S64})
    .clampScalar(0, S16, S64)
    .scalarize(0);
} else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
  getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
    .legalFor({S32, S64})
    .clampScalar(0, S32, S64)
    .scalarize(0);
} else {
  getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT})
    .legalFor({S32})
    .customFor({S64})
    .clampScalar(0, S32, S64)
    .scalarize(0);
}

getActionDefinitionsBuilder({G_PTR_ADD, G_PTR_MASK})
  .scalarize(0)
  .alwaysLegal();

auto &CmpBuilder =
  getActionDefinitionsBuilder(G_ICMP)
  // The compare output type differs based on the register bank of the output,
  // so make both s1 and s32 legal.
  //
  // Scalar compares producing output in scc will be promoted to s32, as that
  // is the allocatable register type that will be needed for the copy from
  // scc. This will be promoted during RegBankSelect, and we assume something
  // before that won't try to use s32 result types.
  //
  // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
  // bank.
  .legalForCartesianProduct(
    {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
  .legalForCartesianProduct(
    {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
if (ST.has16BitInsts()) {
  CmpBuilder.legalFor({{S1, S16}});
}

CmpBuilder
  .widenScalarToNextPow2(1)
  .clampScalar(1, S32, S64)
  .scalarize(0)
  .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1)));

getActionDefinitionsBuilder(G_FCMP)
  .legalForCartesianProduct({S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase)
  .widenScalarToNextPow2(1)
  .clampScalar(1, S32, S64)
  .scalarize(0);

// FIXME: fpow has a selection pattern that should move to custom lowering.
auto &Exp2Ops = getActionDefinitionsBuilder({G_FEXP2, G_FLOG2});
if (ST.has16BitInsts())
  Exp2Ops.legalFor({S32, S16});
else
  Exp2Ops.legalFor({S32});
Exp2Ops.clampScalar(0, MinScalarFPTy, S32);
Exp2Ops.scalarize(0);

auto &ExpOps = getActionDefinitionsBuilder({G_FEXP, G_FLOG, G_FLOG10, G_FPOW});
if (ST.has16BitInsts())
  ExpOps.customFor({{S32}, {S16}});
else
  ExpOps.customFor({S32});
ExpOps.clampScalar(0, MinScalarFPTy, S32)
      .scalarize(0);

// The 64-bit versions produce 32-bit results, but only on the SALU.
getActionDefinitionsBuilder(G_CTPOP)
  .legalFor({{S32, S32}, {S32, S64}})
  .clampScalar(0, S32, S32)
  .clampScalar(1, S32, S64)
  .scalarize(0)
  .widenScalarToNextPow2(0, 32)
  .widenScalarToNextPow2(1, 32);

// The hardware instructions return a different result on 0 than the generic
// instructions expect. The hardware produces -1, but these produce the
// bitwidth.
getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
  .scalarize(0)
  .clampScalar(0, S32, S32)
  .clampScalar(1, S32, S64)
  .widenScalarToNextPow2(0, 32)
  .widenScalarToNextPow2(1, 32)
  .lower();

// The 64-bit versions produce 32-bit results, but only on the SALU.
getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF})
  .legalFor({{S32, S32}, {S32, S64}})
  .clampScalar(0, S32, S32)
  .clampScalar(1, S32, S64)
  .scalarize(0)
  .widenScalarToNextPow2(0, 32)
  .widenScalarToNextPow2(1, 32);

getActionDefinitionsBuilder(G_BITREVERSE)
  .legalFor({S32})
  .clampScalar(0, S32, S32)
  .scalarize(0);

if (ST.has16BitInsts()) {
  getActionDefinitionsBuilder(G_BSWAP)
    .legalFor({S16, S32, V2S16})
    .clampMaxNumElements(0, S16, 2)
    // FIXME: Fixing non-power-of-2 before clamp is workaround for
    // narrowScalar limitation.
    .widenScalarToNextPow2(0)
    .clampScalar(0, S16, S32)
    .scalarize(0);

  if (ST.hasVOP3PInsts()) {
    getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
      .legalFor({S32, S16, V2S16})
      .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
      .clampMaxNumElements(0, S16, 2)
      .minScalar(0, S16)
      .widenScalarToNextPow2(0)
      .scalarize(0)
      .lower();
  } else {
    getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
      .legalFor({S32, S16})
      .widenScalarToNextPow2(0)
      .minScalar(0, S16)
      .scalarize(0)
      .lower();
  }
} else {
  // TODO: Should have same legality without v_perm_b32
  getActionDefinitionsBuilder(G_BSWAP)
    .legalFor({S32})
    .lowerIf(narrowerThan(0, 32))
    // FIXME: Fixing non-power-of-2 before clamp is workaround for
    // narrowScalar limitation.
    .widenScalarToNextPow2(0)
    .maxScalar(0, S32)
    .scalarize(0)
    .lower();

  getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
    .legalFor({S32})
    .minScalar(0, S32)
    .widenScalarToNextPow2(0)
    .scalarize(0)
    .lower();
}

getActionDefinitionsBuilder(G_INTTOPTR)
  // List the common cases
  .legalForCartesianProduct(AddrSpaces64, {S64})
  .legalForCartesianProduct(AddrSpaces32, {S32})
  .scalarize(0)
  // Accept any address space as long as the size matches
  .legalIf(sameSize(0, 1))
  .widenScalarIf(smallerThan(1, 0),
    [](const LegalityQuery &Query) {
      return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
    })
  .narrowScalarIf(greaterThan(1, 0),
    [](const LegalityQuery &Query) {
      return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
    });

getActionDefinitionsBuilder(G_PTRTOINT)
  // List the common cases
  .legalForCartesianProduct(AddrSpaces64, {S64})
  .legalForCartesianProduct(AddrSpaces32, {S32})
  .scalarize(0)
  // Accept any address space as long as the size matches
  .legalIf(sameSize(0, 1))
  .widenScalarIf(smallerThan(0, 1),
    [](const LegalityQuery &Query) {
      return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
    })
  .narrowScalarIf(
    greaterThan(0, 1),
    [](const LegalityQuery &Query) {
      return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
    });

getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
  .scalarize(0)
  .custom();

// TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
// handle some operations by just promoting the register during
// selection. There are also d16 loads on GFX9+ which preserve the high bits.
auto maxSizeForAddrSpace = [this](unsigned AS, bool IsLoad) -> unsigned {
  switch (AS) {
  // FIXME: Private element size.
  case AMDGPUAS::PRIVATE_ADDRESS:
    return 32;
  // FIXME: Check subtarget
  case AMDGPUAS::LOCAL_ADDRESS:
    return ST.useDS128() ? 128 : 64;

  // Treat constant and global as identical. SMRD loads are sometimes usable
  // for global loads (ideally constant address space should be eliminated)
  // depending on the context. Legality cannot be context dependent, but
  // RegBankSelect can split the load as necessary depending on the pointer
  // register bank/uniformity and if the memory is invariant or not written in
  // a kernel.
  case AMDGPUAS::CONSTANT_ADDRESS:
  case AMDGPUAS::GLOBAL_ADDRESS:
    return IsLoad ? 512 : 128;
  default:
    return 128;
  }
};

const auto needToSplitMemOp = [=](const LegalityQuery &Query,
                                  bool IsLoad) -> bool {
  const LLT DstTy = Query.Types[0];

  // Split vector extloads.
  unsigned MemSize = Query.MMODescrs[0].SizeInBits;
  unsigned Align = Query.MMODescrs[0].AlignInBits;

  if (MemSize < DstTy.getSizeInBits())
    MemSize = std::max(MemSize, Align);

  if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
    return true;

  const LLT PtrTy = Query.Types[1];
  unsigned AS = PtrTy.getAddressSpace();
  if (MemSize > maxSizeForAddrSpace(AS, IsLoad))
    return true;

  // Catch weird sized loads that don't evenly divide into the access sizes
  // TODO: May be able to widen depending on alignment etc.
  unsigned NumRegs = (MemSize + 31) / 32;
  if (NumRegs == 3) {
    if (!ST.hasDwordx3LoadStores())
      return true;
  } else {
    // If the alignment allows, these should have been widened.
    if (!isPowerOf2_32(NumRegs))
      return true;
  }

  if (Align < MemSize) {
    const SITargetLowering *TLI = ST.getTargetLowering();
    return !TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS, Align / 8);
  }

  return false;
};

const auto shouldWidenLoadResult = [=](const LegalityQuery &Query) -> bool {
  unsigned Size = Query.Types[0].getSizeInBits();
  if (isPowerOf2_32(Size))
    return false;

  if (Size == 96 && ST.hasDwordx3LoadStores())
    return false;

  unsigned AddrSpace = Query.Types[1].getAddressSpace();
  if (Size >= maxSizeForAddrSpace(AddrSpace, true))
    return false;

  unsigned Align = Query.MMODescrs[0].AlignInBits;
  unsigned RoundedSize = NextPowerOf2(Size);
  return (Align >= RoundedSize);
};

unsigned GlobalAlign32 = ST.hasUnalignedBufferAccess() ? 0 : 32;
unsigned GlobalAlign16 = ST.hasUnalignedBufferAccess() ? 0 : 16;
unsigned GlobalAlign8 = ST.hasUnalignedBufferAccess() ? 0 : 8;

// TODO: Refine based on subtargets which support unaligned access or 128-bit
// LDS
// TODO: Unsupported flat for SI.

for (unsigned Op : {G_LOAD, G_STORE}) {
  const bool IsStore = Op == G_STORE;

  auto &Actions = getActionDefinitionsBuilder(Op);
  // Whitelist the common cases.
  // TODO: Loads to s16 on gfx9
  Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, 32, GlobalAlign32},
                                    {V2S32, GlobalPtr, 64, GlobalAlign32},
                                    {V4S32, GlobalPtr, 128, GlobalAlign32},
                                    {S128, GlobalPtr, 128, GlobalAlign32},
                                    {S64, GlobalPtr, 64, GlobalAlign32},
                                    {V2S64, GlobalPtr, 128, GlobalAlign32},
                                    {V2S16, GlobalPtr, 32, GlobalAlign32},
                                    {S32, GlobalPtr, 8, GlobalAlign8},
                                    {S32, GlobalPtr, 16, GlobalAlign16},

                                    {S32, LocalPtr, 32, 32},
                                    {S64, LocalPtr, 64, 32},
                                    {V2S32, LocalPtr, 64, 32},
                                    {S32, LocalPtr, 8, 8},
                                    {S32, LocalPtr, 16, 16},
                                    {V2S16, LocalPtr, 32, 32},

                                    {S32, PrivatePtr, 32, 32},
                                    {S32, PrivatePtr, 8, 8},
                                    {S32, PrivatePtr, 16, 16},
                                    {V2S16, PrivatePtr, 32, 32},

                                    {S32, FlatPtr, 32, GlobalAlign32},
                                    {S32, FlatPtr, 16, GlobalAlign16},
                                    {S32, FlatPtr, 8, GlobalAlign8},
                                    {V2S16, FlatPtr, 32, GlobalAlign32},

                                    {S32, ConstantPtr, 32, GlobalAlign32},
                                    {V2S32, ConstantPtr, 64, GlobalAlign32},
                                    {V4S32, ConstantPtr, 128, GlobalAlign32},
                                    {S64, ConstantPtr, 64, GlobalAlign32},
                                    {S128, ConstantPtr, 128, GlobalAlign32},
                                    {V2S32, ConstantPtr, 32, GlobalAlign32}});
  Actions
      .customIf(typeIs(1, Constant32Ptr))
      // Widen suitably aligned loads by loading extra elements.
      .moreElementsIf([=](const LegalityQuery &Query) {
          const LLT Ty = Query.Types[0];
          return Op == G_LOAD && Ty.isVector() &&
                 shouldWidenLoadResult(Query);
        }, moreElementsToNextPow2(0))
      .widenScalarIf([=](const LegalityQuery &Query) {
          const LLT Ty = Query.Types[0];
          return Op == G_LOAD && !Ty.isVector() &&
                 shouldWidenLoadResult(Query);
        }, widenScalarOrEltToNextPow2(0))
      .narrowScalarIf(
          [=](const LegalityQuery &Query) -> bool {
            return !Query.Types[0].isVector() &&
                   needToSplitMemOp(Query, Op == G_LOAD);
          },
          [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
            const LLT DstTy = Query.Types[0];
            const LLT PtrTy = Query.Types[1];

            const unsigned DstSize = DstTy.getSizeInBits();
            unsigned MemSize = Query.MMODescrs[0].SizeInBits;

            // Split extloads.
            if (DstSize > MemSize)
              return std::make_pair(0, LLT::scalar(MemSize));

            if (!isPowerOf2_32(DstSize)) {
              // We're probably decomposing an odd sized store. Try to split
              // to the widest type. TODO: Account for alignment. As-is it
              // should be OK, since the new parts will be further legalized.
              unsigned FloorSize = PowerOf2Floor(DstSize);
              return std::make_pair(0, LLT::scalar(FloorSize));
            }

            if (DstSize > 32 && (DstSize % 32 != 0)) {
              // FIXME: Need a way to specify non-extload of larger size if
              // suitably aligned.
              return std::make_pair(0, LLT::scalar(32 * (DstSize / 32)));
            }

            unsigned MaxSize = maxSizeForAddrSpace(PtrTy.getAddressSpace(),
                                                   Op == G_LOAD);
            if (MemSize > MaxSize)
              return std::make_pair(0, LLT::scalar(MaxSize));

            unsigned Align = Query.MMODescrs[0].AlignInBits;
            return std::make_pair(0, LLT::scalar(Align));
          })
      .fewerElementsIf(
          [=](const LegalityQuery &Query) -> bool {
            return Query.Types[0].isVector() &&
                   needToSplitMemOp(Query, Op == G_LOAD);
          },
          [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
            const LLT DstTy = Query.Types[0];
            const LLT PtrTy = Query.Types[1];

            LLT EltTy = DstTy.getElementType();
            unsigned MaxSize = maxSizeForAddrSpace(PtrTy.getAddressSpace(),
                                                   Op == G_LOAD);

            // FIXME: Handle widened to power of 2 results better. This ends
            // up scalarizing.
            // FIXME: 3 element stores scalarized on SI

            // Split if it's too large for the address space.
            if (Query.MMODescrs[0].SizeInBits > MaxSize) {
              unsigned NumElts = DstTy.getNumElements();
              unsigned EltSize = EltTy.getSizeInBits();

              if (MaxSize % EltSize == 0) {
                return std::make_pair(
                  0, LLT::scalarOrVector(MaxSize / EltSize, EltTy));
              }

              unsigned NumPieces = Query.MMODescrs[0].SizeInBits / MaxSize;

              // FIXME: Refine when odd breakdowns handled
              // The scalars will need to be re-legalized.
              if (NumPieces == 1 || NumPieces >= NumElts ||
                  NumElts % NumPieces != 0)
                return std::make_pair(0, EltTy);

              return std::make_pair(0,
                                    LLT::vector(NumElts / NumPieces, EltTy));
            }

            // FIXME: We could probably handle weird extending loads better.
            unsigned MemSize = Query.MMODescrs[0].SizeInBits;
            if (DstTy.getSizeInBits() > MemSize)
              return std::make_pair(0, EltTy);

            unsigned EltSize = EltTy.getSizeInBits();
            unsigned DstSize = DstTy.getSizeInBits();
            if (!isPowerOf2_32(DstSize)) {
              // We're probably decomposing an odd sized store. Try to split
              // to the widest type. TODO: Account for alignment. As-is it
              // should be OK, since the new parts will be further legalized.
              unsigned FloorSize = PowerOf2Floor(DstSize);
              return std::make_pair(
                0, LLT::scalarOrVector(FloorSize / EltSize, EltTy));
            }

            // Need to split because of alignment.
            unsigned Align = Query.MMODescrs[0].AlignInBits;
            if (EltSize > Align &&
                (EltSize / Align < DstTy.getNumElements())) {
              return std::make_pair(0, LLT::vector(EltSize / Align, EltTy));
            }

            // May need relegalization for the scalars.
            return std::make_pair(0, EltTy);
          })
      .minScalar(0, S32);

  if (IsStore)
    Actions.narrowScalarIf(isWideScalarTruncStore(0), changeTo(0, S32));

  // TODO: Need a bitcast lower option?
  Actions
      .legalIf([=](const LegalityQuery &Query) {
        const LLT Ty0 = Query.Types[0];
        unsigned Size = Ty0.getSizeInBits();
        unsigned MemSize = Query.MMODescrs[0].SizeInBits;
        unsigned Align = Query.MMODescrs[0].AlignInBits;

        // FIXME: Widening store from alignment not valid.
        if (MemSize < Size)
          MemSize = std::max(MemSize, Align);

        // No extending vector loads.
        if (Size > MemSize && Ty0.isVector())
          return false;

        switch (MemSize) {
        case 8:
        case 16:
          return Size == 32;
        case 32:
        case 64:
        case 128:
          return true;
        case 96:
          return ST.hasDwordx3LoadStores();
        case 256:
        case 512:
          return true;
        default:
          return false;
        }
      })
      .widenScalarToNextPow2(0)
      .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0));
}

auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
                     .legalForTypesWithMemDesc({{S32, GlobalPtr, 8, 8},
                                                {S32, GlobalPtr, 16, 2 * 8},
                                                {S32, LocalPtr, 8, 8},
                                                {S32, LocalPtr, 16, 16},
                                                {S32, PrivatePtr, 8, 8},
                                                {S32, PrivatePtr, 16, 16},
                                                {S32, ConstantPtr, 8, 8},
                                                {S32, ConstantPtr, 16, 2 * 8}});
if (ST.hasFlatAddressSpace()) {
  ExtLoads.legalForTypesWithMemDesc(
      {{S32, FlatPtr, 8, 8}, {S32, FlatPtr, 16, 16}});
}

ExtLoads.clampScalar(0, S32, S32)
        .widenScalarToNextPow2(0)
        .unsupportedIfMemSizeNotPow2()
        .lower();

auto &Atomics = getActionDefinitionsBuilder(
  {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
   G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
   G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
   G_ATOMICRMW_UMIN})
  .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
             {S64, GlobalPtr}, {S64, LocalPtr}});
if (ST.hasFlatAddressSpace()) {
  Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
}

getActionDefinitionsBuilder(G_ATOMICRMW_FADD)
  .legalFor({{S32, LocalPtr}});

// BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
// demarshalling
getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
  .customFor({{S32, GlobalPtr}, {S64, GlobalPtr},
              {S32, FlatPtr}, {S64, FlatPtr}})
  .legalFor({{S32, LocalPtr}, {S64, LocalPtr},
             {S32, RegionPtr}, {S64, RegionPtr}});
// TODO: Pointer types, any 32-bit or 64-bit vector

// Condition should be s32 for scalar, s1 for vector.
getActionDefinitionsBuilder(G_SELECT)
  .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16,
        GlobalPtr, LocalPtr, FlatPtr, PrivatePtr,
        LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1, S32})
  .clampScalar(0, S16, S64)
  .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
  .fewerElementsIf(numElementsNotEven(0), scalarize(0))
  .scalarize(1)
  .clampMaxNumElements(0, S32, 2)
  .clampMaxNumElements(0, LocalPtr, 2)
  .clampMaxNumElements(0, PrivatePtr, 2)
  .scalarize(0)
  .widenScalarToNextPow2(0)
  .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));

// TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
// be more flexible with the shift amount type.
auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
  .legalFor({{S32, S32}, {S64, S32}});
if (ST.has16BitInsts()) {
  if (ST.hasVOP3PInsts()) {
    Shifts.legalFor({{S16, S32}, {S16, S16}, {V2S16, V2S16}})
          .clampMaxNumElements(0, S16, 2);
  } else
    Shifts.legalFor({{S16, S32}, {S16, S16}});

  // TODO: Support 16-bit shift amounts
  Shifts.clampScalar(1, S32, S32);
  Shifts.clampScalar(0, S16, S64);
  Shifts.widenScalarToNextPow2(0, 16);
} else {
  // Make sure we legalize the shift amount type first, as the general
  // expansion for the shifted type will produce much worse code if it hasn't
  // been truncated already.
  Shifts.clampScalar(1, S32, S32);
  Shifts.clampScalar(0, S32, S64);
  Shifts.widenScalarToNextPow2(0, 32);
}
Shifts.scalarize(0);

for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
  unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
  unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
  unsigned IdxTypeIdx = 2;

  getActionDefinitionsBuilder(Op)
    .customIf([=](const LegalityQuery &Query) {
        const LLT EltTy = Query.Types[EltTypeIdx];
        const LLT VecTy = Query.Types[VecTypeIdx];
        const LLT IdxTy = Query.Types[IdxTypeIdx];
        return (EltTy.getSizeInBits() == 16 ||
                EltTy.getSizeInBits() % 32 == 0) &&
               VecTy.getSizeInBits() % 32 == 0 &&
               VecTy.getSizeInBits() <= 1024 &&
               IdxTy.getSizeInBits() == 32;
      })
    .clampScalar(EltTypeIdx, S32, S64)
    .clampScalar(VecTypeIdx, S32, S64)
    .clampScalar(IdxTypeIdx, S32, S32);
}

getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
  .unsupportedIf([=](const LegalityQuery &Query) {
      const LLT &EltTy = Query.Types[1].getElementType();
      return Query.Types[0] != EltTy;
    });

for (unsigned Op : {G_EXTRACT, G_INSERT}) {
  unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
  unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;

  // FIXME: Doesn't handle extract of illegal sizes.
  getActionDefinitionsBuilder(Op)
    .lowerIf(all(typeIs(LitTyIdx, S16), sizeIs(BigTyIdx, 32)))
    // FIXME: Multiples of 16 should not be legal.
    .legalIf([=](const LegalityQuery &Query) {
        const LLT BigTy = Query.Types[BigTyIdx];
        const LLT LitTy = Query.Types[LitTyIdx];
        return (BigTy.getSizeInBits() % 32 == 0) &&
               (LitTy.getSizeInBits() % 16 == 0);
      })
    .widenScalarIf(
      [=](const LegalityQuery &Query) {
        const LLT BigTy = Query.Types[BigTyIdx];
        return (BigTy.getScalarSizeInBits() < 16);
      },
      LegalizeMutations::widenScalarOrEltToNextPow2(BigTyIdx, 16))
    .widenScalarIf(
      [=](const LegalityQuery &Query) {
        const LLT LitTy = Query.Types[LitTyIdx];
        return (LitTy.getScalarSizeInBits() < 16);
      },
      LegalizeMutations::widenScalarOrEltToNextPow2(LitTyIdx, 16))
    .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
    .widenScalarToNextPow2(BigTyIdx, 32);

}

auto &BuildVector = getActionDefinitionsBuilder(G_BUILD_VECTOR)
  .legalForCartesianProduct(AllS32Vectors, {S32})
  .legalForCartesianProduct(AllS64Vectors, {S64})
  .clampNumElements(0, V16S32, V32S32)
  .clampNumElements(0, V2S64, V16S64)
  .fewerElementsIf(isWideVec16(0), changeTo(0, V2S16));

if (ST.hasScalarPackInsts()) {
  BuildVector
    // FIXME: Should probably widen s1 vectors straight to s32
    .minScalarOrElt(0, S16)
    // Widen source elements and produce a G_BUILD_VECTOR_TRUNC
    .minScalar(1, S32);

  getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
    .legalFor({V2S16, S32})
    .lower();
  BuildVector.minScalarOrElt(0, S32);
} else {
  BuildVector.customFor({V2S16, S16});
  BuildVector.minScalarOrElt(0, S32);

  getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
    .customFor({V2S16, S32})
    .lower();
}

BuildVector.legalIf(isRegisterType(0));

// FIXME: Clamp maximum size
getActionDefinitionsBuilder(G_CONCAT_VECTORS)
  .legalIf(isRegisterType(0));

// TODO: Don't fully scalarize v2s16 pieces? Or combine out thosse
// pre-legalize.
if (ST.hasVOP3PInsts()) {
  getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
    .customFor({V2S16, V2S16})
    .lower();
} else
  getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();

// Merge/Unmerge
for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
  unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
  unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;

  auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
    const LLT &Ty = Query.Types[TypeIdx];
    if (Ty.isVector()) {
      const LLT &EltTy = Ty.getElementType();
      if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64)
        return true;
      if (!isPowerOf2_32(EltTy.getSizeInBits()))
        return true;
    }
    return false;
  };

  auto &Builder = getActionDefinitionsBuilder(Op)
    // Try to widen to s16 first for small types.
    // TODO: Only do this on targets with legal s16 shifts
    .minScalarOrEltIf(narrowerThan(LitTyIdx, 16), LitTyIdx, S16)

    .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
    .lowerFor({{S16, V2S16}})
    .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
    .fewerElementsIf(all(typeIs(0, S16), vectorWiderThan(1, 32),
                         elementTypeIs(1, S16)),
                     changeTo(1, V2S16))
    // Clamp the little scalar to s8-s256 and make it a power of 2. It's not
    // worth considering the multiples of 64 since 2*192 and 2*384 are not
    // valid.
    .clampScalar(LitTyIdx, S32, S256)
    .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
    // Break up vectors with weird elements into scalars
    .fewerElementsIf(
      [=](const LegalityQuery &Query) { return notValidElt(Query, 0); },
      scalarize(0))
    .fewerElementsIf(
      [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
      scalarize(1))
    .clampScalar(BigTyIdx, S32, S1024);

  if (Op == G_MERGE_VALUES) {
    Builder.widenScalarIf(
      // TODO: Use 16-bit shifts if legal for 8-bit values?
      [=](const LegalityQuery &Query) {
        const LLT Ty = Query.Types[LitTyIdx];
        return Ty.getSizeInBits() < 32;
      },
      changeTo(LitTyIdx, S32));
  }

  Builder.widenScalarIf(
    [=](const LegalityQuery &Query) {
      const LLT Ty = Query.Types[BigTyIdx];
      return !isPowerOf2_32(Ty.getSizeInBits()) &&
        Ty.getSizeInBits() % 16 != 0;
    },
    [=](const LegalityQuery &Query) {
      // Pick the next power of 2, or a multiple of 64 over 128.
      // Whichever is smaller.
      const LLT &Ty = Query.Types[BigTyIdx];
      unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
      if (NewSizeInBits >= 256) {
        unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
        if (RoundedTo < NewSizeInBits)
          NewSizeInBits = RoundedTo;
      }
      return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits));
    })
    .legalIf([=](const LegalityQuery &Query) {
        const LLT &BigTy = Query.Types[BigTyIdx];
        const LLT &LitTy = Query.Types[LitTyIdx];

        if (BigTy.isVector() && BigTy.getSizeInBits() < 32)
          return false;
        if (LitTy.isVector() && LitTy.getSizeInBits() < 32)
          return false;

        return BigTy.getSizeInBits() % 16 == 0 &&
               LitTy.getSizeInBits() % 16 == 0 &&
               BigTy.getSizeInBits() <= 1024;
      })
    // Any vectors left are the wrong size. Scalarize them.
    .scalarize(0)
    .scalarize(1);
}

// S64 is only legal on SALU, and needs to be broken into 32-bit elements in
// RegBankSelect.
auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
  .legalFor({{S32}, {S64}});

if (ST.hasVOP3PInsts()) {
  SextInReg.lowerFor({{V2S16}})
    // Prefer to reduce vector widths for 16-bit vectors before lowering, to
    // get more vector shift opportunities, since we'll get those when
    // expanded.
    .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16));
} else if (ST.has16BitInsts()) {
  SextInReg.lowerFor({{S32}, {S64}, {S16}});
} else {
  // Prefer to promote to s32 before lowering if we don't have 16-bit
  // shifts. This avoid a lot of intermediate truncate and extend operations.
  SextInReg.lowerFor({{S32}, {S64}});
}

SextInReg
  .scalarize(0)
  .clampScalar(0, S32, S64)
  .lower();

getActionDefinitionsBuilder(G_READCYCLECOUNTER)
  .legalFor({S64});

getActionDefinitionsBuilder({
    // TODO: Verify V_BFI_B32 is generated from expanded bit ops
    G_FCOPYSIGN,

    G_ATOMIC_CMPXCHG_WITH_SUCCESS,
    G_READ_REGISTER,
    G_WRITE_REGISTER,

    G_SADDO, G_SSUBO,

     // TODO: Implement
    G_FMINIMUM, G_FMAXIMUM
  }).lower();

getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
      G_DYN_STACKALLOC, G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
      G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
  .unsupported();

computeTables();
verify(*ST.getInstrInfo());
1320}

1322bool AMDGPULegalizerInfo::legalizeCustom(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B,
                                       GISelChangeObserver &Observer) const {
switch (MI.getOpcode()) {
case TargetOpcode::G_ADDRSPACE_CAST:
  return legalizeAddrSpaceCast(MI, MRI, B);
case TargetOpcode::G_FRINT:
  return legalizeFrint(MI, MRI, B);
case TargetOpcode::G_FCEIL:
  return legalizeFceil(MI, MRI, B);
case TargetOpcode::G_INTRINSIC_TRUNC:
  return legalizeIntrinsicTrunc(MI, MRI, B);
case TargetOpcode::G_SITOFP:
  return legalizeITOFP(MI, MRI, B, true);
case TargetOpcode::G_UITOFP:
  return legalizeITOFP(MI, MRI, B, false);
case TargetOpcode::G_FPTOSI:
  return legalizeFPTOI(MI, MRI, B, true);
case TargetOpcode::G_FPTOUI:
  return legalizeFPTOI(MI, MRI, B, false);
case TargetOpcode::G_FMINNUM:
case TargetOpcode::G_FMAXNUM:
case TargetOpcode::G_FMINNUM_IEEE:
case TargetOpcode::G_FMAXNUM_IEEE:
  return legalizeMinNumMaxNum(MI, MRI, B);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
  return legalizeExtractVectorElt(MI, MRI, B);
case TargetOpcode::G_INSERT_VECTOR_ELT:
  return legalizeInsertVectorElt(MI, MRI, B);
case TargetOpcode::G_SHUFFLE_VECTOR:
  return legalizeShuffleVector(MI, MRI, B);
case TargetOpcode::G_FSIN:
case TargetOpcode::G_FCOS:
  return legalizeSinCos(MI, MRI, B);
case TargetOpcode::G_GLOBAL_VALUE:
  return legalizeGlobalValue(MI, MRI, B);
case TargetOpcode::G_LOAD:
  return legalizeLoad(MI, MRI, B, Observer);
case TargetOpcode::G_FMAD:
  return legalizeFMad(MI, MRI, B);
case TargetOpcode::G_FDIV:
  return legalizeFDIV(MI, MRI, B);
case TargetOpcode::G_UDIV:
case TargetOpcode::G_UREM:
  return legalizeUDIV_UREM(MI, MRI, B);
case TargetOpcode::G_SDIV:
case TargetOpcode::G_SREM:
  return legalizeSDIV_SREM(MI, MRI, B);
case TargetOpcode::G_ATOMIC_CMPXCHG:
  return legalizeAtomicCmpXChg(MI, MRI, B);
case TargetOpcode::G_FLOG:
  return legalizeFlog(MI, B, 1.0f / numbers::log2ef);
case TargetOpcode::G_FLOG10:
  return legalizeFlog(MI, B, numbers::ln2f / numbers::ln10f);
case TargetOpcode::G_FEXP:
  return legalizeFExp(MI, B);
case TargetOpcode::G_FPOW:
  return legalizeFPow(MI, B);
case TargetOpcode::G_FFLOOR:
  return legalizeFFloor(MI, MRI, B);
case TargetOpcode::G_BUILD_VECTOR:
  return legalizeBuildVector(MI, MRI, B);
default:
  return false;
}

llvm_unreachable("expected switch to return")::llvm::llvm_unreachable_internal("expected switch to return"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1389);
1390}

1392Register AMDGPULegalizerInfo::getSegmentAperture(
unsigned AS,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const LLT S32 = LLT::scalar(32);

assert(AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS)((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS
) ? static_cast<void> (0) : __assert_fail ("AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1400, __PRETTY_FUNCTION__));

if (ST.hasApertureRegs()) {
  // FIXME: Use inline constants (src_{shared, private}_base) instead of
  // getreg.
  unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ?
      AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE :
      AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE;
  unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ?
      AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE :
      AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE;
  unsigned Encoding =
      AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ |
      Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ |
      WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_;

  Register GetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);

  B.buildInstr(AMDGPU::S_GETREG_B32)
    .addDef(GetReg)
    .addImm(Encoding);
  MRI.setType(GetReg, S32);

  auto ShiftAmt = B.buildConstant(S32, WidthM1 + 1);
  return B.buildShl(S32, GetReg, ShiftAmt).getReg(0);
}

Register QueuePtr = MRI.createGenericVirtualRegister(
  LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));

const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
if (!loadInputValue(QueuePtr, B, &MFI->getArgInfo().QueuePtr))
  return Register();

// Offset into amd_queue_t for group_segment_aperture_base_hi /
// private_segment_aperture_base_hi.
uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;

// TODO: can we be smarter about machine pointer info?
MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
MachineMemOperand *MMO = MF.getMachineMemOperand(
  PtrInfo,
  MachineMemOperand::MOLoad |
  MachineMemOperand::MODereferenceable |
  MachineMemOperand::MOInvariant,
  4,
  MinAlign(64, StructOffset));

Register LoadAddr;

B.materializePtrAdd(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset);
return B.buildLoad(S32, LoadAddr, *MMO).getReg(0);
1452}

1454bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();

B.setInstr(MI);

const LLT S32 = LLT::scalar(32);
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();

LLT DstTy = MRI.getType(Dst);
LLT SrcTy = MRI.getType(Src);
unsigned DestAS = DstTy.getAddressSpace();
unsigned SrcAS = SrcTy.getAddressSpace();

// TODO: Avoid reloading from the queue ptr for each cast, or at least each
// vector element.
assert(!DstTy.isVector())((!DstTy.isVector()) ? static_cast<void> (0) : __assert_fail
 ("!DstTy.isVector()", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1472, __PRETTY_FUNCTION__));

const AMDGPUTargetMachine &TM
  = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());

const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
if (ST.getTargetLowering()->isNoopAddrSpaceCast(SrcAS, DestAS)) {
  MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
  return true;
}

if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
  // Truncate.
  B.buildExtract(Dst, Src, 0);
  MI.eraseFromParent();
  return true;
}

if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  uint32_t AddrHiVal = Info->get32BitAddressHighBits();

  // FIXME: This is a bit ugly due to creating a merge of 2 pointers to
  // another. Merge operands are required to be the same type, but creating an
  // extra ptrtoint would be kind of pointless.
  auto HighAddr = B.buildConstant(
    LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS_32BIT, 32), AddrHiVal);
  B.buildMerge(Dst, {Src, HighAddr});
  MI.eraseFromParent();
  return true;
}

if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
  assert(DestAS == AMDGPUAS::LOCAL_ADDRESS ||((DestAS == AMDGPUAS::LOCAL_ADDRESS || DestAS == AMDGPUAS::PRIVATE_ADDRESS
) ? static_cast<void> (0) : __assert_fail ("DestAS == AMDGPUAS::LOCAL_ADDRESS || DestAS == AMDGPUAS::PRIVATE_ADDRESS"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1506, __PRETTY_FUNCTION__))
         DestAS == AMDGPUAS::PRIVATE_ADDRESS)((DestAS == AMDGPUAS::LOCAL_ADDRESS || DestAS == AMDGPUAS::PRIVATE_ADDRESS
) ? static_cast<void> (0) : __assert_fail ("DestAS == AMDGPUAS::LOCAL_ADDRESS || DestAS == AMDGPUAS::PRIVATE_ADDRESS"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1506, __PRETTY_FUNCTION__));
  unsigned NullVal = TM.getNullPointerValue(DestAS);

  auto SegmentNull = B.buildConstant(DstTy, NullVal);
  auto FlatNull = B.buildConstant(SrcTy, 0);

  // Extract low 32-bits of the pointer.
  auto PtrLo32 = B.buildExtract(DstTy, Src, 0);

  auto CmpRes =
      B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0));
  B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));

  MI.eraseFromParent();
  return true;
}

if (SrcAS != AMDGPUAS::LOCAL_ADDRESS && SrcAS != AMDGPUAS::PRIVATE_ADDRESS)
  return false;

if (!ST.hasFlatAddressSpace())
  return false;

auto SegmentNull =
    B.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
auto FlatNull =
    B.buildConstant(DstTy, TM.getNullPointerValue(DestAS));

Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
if (!ApertureReg.isValid())
  return false;

auto CmpRes =
    B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, SegmentNull.getReg(0));

// Coerce the type of the low half of the result so we can use merge_values.
Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);

// TODO: Should we allow mismatched types but matching sizes in merges to
// avoid the ptrtoint?
auto BuildPtr = B.buildMerge(DstTy, {SrcAsInt, ApertureReg});
B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);

MI.eraseFromParent();
return true;
1551}

1553bool AMDGPULegalizerInfo::legalizeFrint(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
B.setInstr(MI);

Register Src = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(Src);
assert(Ty.isScalar() && Ty.getSizeInBits() == 64)((Ty.isScalar() && Ty.getSizeInBits() == 64) ? static_cast
<void> (0) : __assert_fail ("Ty.isScalar() && Ty.getSizeInBits() == 64"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1560, __PRETTY_FUNCTION__));

APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");

auto C1 = B.buildFConstant(Ty, C1Val);
auto CopySign = B.buildFCopysign(Ty, C1, Src);

// TODO: Should this propagate fast-math-flags?
auto Tmp1 = B.buildFAdd(Ty, Src, CopySign);
auto Tmp2 = B.buildFSub(Ty, Tmp1, CopySign);

auto C2 = B.buildFConstant(Ty, C2Val);
auto Fabs = B.buildFAbs(Ty, Src);

auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
return true;
1578}

1580bool AMDGPULegalizerInfo::legalizeFceil(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
B.setInstr(MI);

const LLT S1 = LLT::scalar(1);
const LLT S64 = LLT::scalar(64);

Register Src = MI.getOperand(1).getReg();
assert(MRI.getType(Src) == S64)((MRI.getType(Src) == S64) ? static_cast<void> (0) : __assert_fail
 ("MRI.getType(Src) == S64", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1589, __PRETTY_FUNCTION__));

// result = trunc(src)
// if (src > 0.0 && src != result)
//   result += 1.0

auto Trunc = B.buildIntrinsicTrunc(S64, Src);

const auto Zero = B.buildFConstant(S64, 0.0);
const auto One = B.buildFConstant(S64, 1.0);
auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
auto And = B.buildAnd(S1, Lt0, NeTrunc);
auto Add = B.buildSelect(S64, And, One, Zero);

// TODO: Should this propagate fast-math-flags?
B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
return true;
1607}

1609static MachineInstrBuilder extractF64Exponent(unsigned Hi,
                                            MachineIRBuilder &B) {
const unsigned FractBits = 52;
const unsigned ExpBits = 11;
LLT S32 = LLT::scalar(32);

auto Const0 = B.buildConstant(S32, FractBits - 32);
auto Const1 = B.buildConstant(S32, ExpBits);

auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {S32}, false)
  .addUse(Const0.getReg(0))
  .addUse(Const1.getReg(0));

return B.buildSub(S32, ExpPart, B.buildConstant(S32, 1023));
1623}

1625bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
B.setInstr(MI);

const LLT S1 = LLT::scalar(1);
const LLT S32 = LLT::scalar(32);
const LLT S64 = LLT::scalar(64);

Register Src = MI.getOperand(1).getReg();
assert(MRI.getType(Src) == S64)((MRI.getType(Src) == S64) ? static_cast<void> (0) : __assert_fail
 ("MRI.getType(Src) == S64", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1635, __PRETTY_FUNCTION__));

// TODO: Should this use extract since the low half is unused?
auto Unmerge = B.buildUnmerge({S32, S32}, Src);
Register Hi = Unmerge.getReg(1);

// Extract the upper half, since this is where we will find the sign and
// exponent.
auto Exp = extractF64Exponent(Hi, B);

const unsigned FractBits = 52;

// Extract the sign bit.
const auto SignBitMask = B.buildConstant(S32, UINT32_C(1)1U << 31);
auto SignBit = B.buildAnd(S32, Hi, SignBitMask);

const auto FractMask = B.buildConstant(S64, (UINT64_C(1)1UL << FractBits) - 1);

const auto Zero32 = B.buildConstant(S32, 0);

// Extend back to 64-bits.
auto SignBit64 = B.buildMerge(S64, {Zero32, SignBit});

auto Shr = B.buildAShr(S64, FractMask, Exp);
auto Not = B.buildNot(S64, Shr);
auto Tmp0 = B.buildAnd(S64, Src, Not);
auto FiftyOne = B.buildConstant(S32, FractBits - 1);

auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);

auto Tmp1 = B.buildSelect(S64, ExpLt0, SignBit64, Tmp0);
B.buildSelect(MI.getOperand(0).getReg(), ExpGt51, Src, Tmp1);
return true;
1669}

1671bool AMDGPULegalizerInfo::legalizeITOFP(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, bool Signed) const {
B.setInstr(MI);

Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();

const LLT S64 = LLT::scalar(64);
const LLT S32 = LLT::scalar(32);

assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64)((MRI.getType(Src) == S64 && MRI.getType(Dst) == S64)
 ? static_cast<void> (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S64"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1682, __PRETTY_FUNCTION__));

auto Unmerge = B.buildUnmerge({S32, S32}, Src);

auto CvtHi = Signed ?
  B.buildSITOFP(S64, Unmerge.getReg(1)) :
  B.buildUITOFP(S64, Unmerge.getReg(1));

auto CvtLo = B.buildUITOFP(S64, Unmerge.getReg(0));

auto ThirtyTwo = B.buildConstant(S32, 32);
auto LdExp = B.buildIntrinsic(Intrinsic::amdgcn_ldexp, {S64}, false)
  .addUse(CvtHi.getReg(0))
  .addUse(ThirtyTwo.getReg(0));

// TODO: Should this propagate fast-math-flags?
B.buildFAdd(Dst, LdExp, CvtLo);
MI.eraseFromParent();
return true;
1701}

1703// TODO: Copied from DAG implementation. Verify logic and document how this
1704// actually works.
1705bool AMDGPULegalizerInfo::legalizeFPTOI(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, bool Signed) const {
B.setInstr(MI);

Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();

const LLT S64 = LLT::scalar(64);
const LLT S32 = LLT::scalar(32);

assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64)((MRI.getType(Src) == S64 && MRI.getType(Dst) == S64)
 ? static_cast<void> (0) : __assert_fail ("MRI.getType(Src) == S64 && MRI.getType(Dst) == S64"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1716, __PRETTY_FUNCTION__));

unsigned Flags = MI.getFlags();

auto Trunc = B.buildIntrinsicTrunc(S64, Src, Flags);
auto K0 = B.buildFConstant(S64, BitsToDouble(UINT64_C(0x3df0000000000000)0x3df0000000000000UL));
auto K1 = B.buildFConstant(S64, BitsToDouble(UINT64_C(0xc1f0000000000000)0xc1f0000000000000UL));

auto Mul = B.buildFMul(S64, Trunc, K0, Flags);
auto FloorMul = B.buildFFloor(S64, Mul, Flags);
auto Fma = B.buildFMA(S64, FloorMul, K1, Trunc, Flags);

auto Hi = Signed ?
  B.buildFPTOSI(S32, FloorMul) :
  B.buildFPTOUI(S32, FloorMul);
auto Lo = B.buildFPTOUI(S32, Fma);

B.buildMerge(Dst, { Lo, Hi });
MI.eraseFromParent();

return true;
1737}

1739bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

const bool IsIEEEOp = MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE ||
                      MI.getOpcode() == AMDGPU::G_FMAXNUM_IEEE;

// With ieee_mode disabled, the instructions have the correct behavior
// already for G_FMINNUM/G_FMAXNUM
if (!MFI->getMode().IEEE)
  return !IsIEEEOp;

if (IsIEEEOp)
  return true;

MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
HelperBuilder.setInstr(MI);
return Helper.lowerFMinNumMaxNum(MI) == LegalizerHelper::Legalized;
1761}

1763bool AMDGPULegalizerInfo::legalizeExtractVectorElt(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// TODO: Should move some of this into LegalizerHelper.

// TODO: Promote dynamic indexing of s16 to s32

// FIXME: Artifact combiner probably should have replaced the truncated
// constant before this, so we shouldn't need
// getConstantVRegValWithLookThrough.
Optional<ValueAndVReg> IdxVal = getConstantVRegValWithLookThrough(
  MI.getOperand(2).getReg(), MRI);
if (!IdxVal) // Dynamic case will be selected to register indexing.
  return true;

Register Dst = MI.getOperand(0).getReg();
Register Vec = MI.getOperand(1).getReg();

LLT VecTy = MRI.getType(Vec);
LLT EltTy = VecTy.getElementType();
assert(EltTy == MRI.getType(Dst))((EltTy == MRI.getType(Dst)) ? static_cast<void> (0) : __assert_fail
 ("EltTy == MRI.getType(Dst)", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1783, __PRETTY_FUNCTION__));

B.setInstr(MI);

if (IdxVal->Value < VecTy.getNumElements())
  B.buildExtract(Dst, Vec, IdxVal->Value * EltTy.getSizeInBits());
else
  B.buildUndef(Dst);

MI.eraseFromParent();
return true;
1794}

1796bool AMDGPULegalizerInfo::legalizeInsertVectorElt(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// TODO: Should move some of this into LegalizerHelper.

// TODO: Promote dynamic indexing of s16 to s32

// FIXME: Artifact combiner probably should have replaced the truncated
// constant before this, so we shouldn't need
// getConstantVRegValWithLookThrough.
Optional<ValueAndVReg> IdxVal = getConstantVRegValWithLookThrough(
  MI.getOperand(3).getReg(), MRI);
if (!IdxVal) // Dynamic case will be selected to register indexing.
  return true;

Register Dst = MI.getOperand(0).getReg();
Register Vec = MI.getOperand(1).getReg();
Register Ins = MI.getOperand(2).getReg();

LLT VecTy = MRI.getType(Vec);
LLT EltTy = VecTy.getElementType();
assert(EltTy == MRI.getType(Ins))((EltTy == MRI.getType(Ins)) ? static_cast<void> (0) : __assert_fail
 ("EltTy == MRI.getType(Ins)", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 1817, __PRETTY_FUNCTION__));

B.setInstr(MI);

if (IdxVal->Value < VecTy.getNumElements())
  B.buildInsert(Dst, Vec, Ins, IdxVal->Value * EltTy.getSizeInBits());
else
  B.buildUndef(Dst);

MI.eraseFromParent();
return true;
1828}

1830bool AMDGPULegalizerInfo::legalizeShuffleVector(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT V2S16 = LLT::vector(2, 16);

Register Dst = MI.getOperand(0).getReg();
Register Src0 = MI.getOperand(1).getReg();
LLT DstTy = MRI.getType(Dst);
LLT SrcTy = MRI.getType(Src0);

if (SrcTy == V2S16 && DstTy == V2S16 &&
    AMDGPU::isLegalVOP3PShuffleMask(MI.getOperand(3).getShuffleMask()))
  return true;

MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(B.getMF(), DummyObserver, HelperBuilder);
HelperBuilder.setInstr(MI);
return Helper.lowerShuffleVector(MI) == LegalizerHelper::Legalized;
1849}

1851bool AMDGPULegalizerInfo::legalizeSinCos(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
B.setInstr(MI);

Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(DstReg);
unsigned Flags = MI.getFlags();

Register TrigVal;
auto OneOver2Pi = B.buildFConstant(Ty, 0.5 / M_PI3.14159265358979323846);
if (ST.hasTrigReducedRange()) {
  auto MulVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
  TrigVal = B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty}, false)
    .addUse(MulVal.getReg(0))
    .setMIFlags(Flags).getReg(0);
} else
  TrigVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);

Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
  Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
B.buildIntrinsic(TrigIntrin, makeArrayRef<Register>(DstReg), false)
  .addUse(TrigVal)
  .setMIFlags(Flags);
MI.eraseFromParent();
return true;
1878}

1880bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(
Register DstReg, LLT PtrTy,
MachineIRBuilder &B, const GlobalValue *GV,
unsigned Offset, unsigned GAFlags) const {
// In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
// to the following code sequence:
//
// For constant address space:
//   s_getpc_b64 s[0:1]
//   s_add_u32 s0, s0, $symbol
//   s_addc_u32 s1, s1, 0
//
//   s_getpc_b64 returns the address of the s_add_u32 instruction and then
//   a fixup or relocation is emitted to replace $symbol with a literal
//   constant, which is a pc-relative offset from the encoding of the $symbol
//   operand to the global variable.
//
// For global address space:
//   s_getpc_b64 s[0:1]
//   s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
//   s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
//
//   s_getpc_b64 returns the address of the s_add_u32 instruction and then
//   fixups or relocations are emitted to replace $symbol@*@lo and
//   $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
//   which is a 64-bit pc-relative offset from the encoding of the $symbol
//   operand to the global variable.
//
// What we want here is an offset from the value returned by s_getpc
// (which is the address of the s_add_u32 instruction) to the global
// variable, but since the encoding of $symbol starts 4 bytes after the start
// of the s_add_u32 instruction, we end up with an offset that is 4 bytes too
// small. This requires us to add 4 to the global variable offset in order to
// compute the correct address.

LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);

Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
  B.getMRI()->createGenericVirtualRegister(ConstPtrTy);

MachineInstrBuilder MIB = B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET)
  .addDef(PCReg);

MIB.addGlobalAddress(GV, Offset + 4, GAFlags);
if (GAFlags == SIInstrInfo::MO_NONE)
  MIB.addImm(0);
else
  MIB.addGlobalAddress(GV, Offset + 4, GAFlags + 1);

B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);

if (PtrTy.getSizeInBits() == 32)
  B.buildExtract(DstReg, PCReg, 0);
return true;
}

1936bool AMDGPULegalizerInfo::legalizeGlobalValue(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register DstReg = MI.getOperand(0).getReg();
LLT Ty = MRI.getType(DstReg);
unsigned AS = Ty.getAddressSpace();

const GlobalValue *GV = MI.getOperand(1).getGlobal();
MachineFunction &MF = B.getMF();
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
B.setInstr(MI);

if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
  if (!MFI->isEntryFunction()) {
    const Function &Fn = MF.getFunction();
    DiagnosticInfoUnsupported BadLDSDecl(
      Fn, "local memory global used by non-kernel function", MI.getDebugLoc());
    Fn.getContext().diagnose(BadLDSDecl);
  }

  // TODO: We could emit code to handle the initialization somewhere.
  if (!AMDGPUTargetLowering::hasDefinedInitializer(GV)) {
    const SITargetLowering *TLI = ST.getTargetLowering();
    if (!TLI->shouldUseLDSConstAddress(GV)) {
      MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
      return true; // Leave in place;
    }

    B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(), *GV));
    MI.eraseFromParent();
    return true;
  }

  const Function &Fn = MF.getFunction();
  DiagnosticInfoUnsupported BadInit(
    Fn, "unsupported initializer for address space", MI.getDebugLoc());
  Fn.getContext().diagnose(BadInit);
  return true;
}

const SITargetLowering *TLI = ST.getTargetLowering();

if (TLI->shouldEmitFixup(GV)) {
  buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
  MI.eraseFromParent();
  return true;
}

if (TLI->shouldEmitPCReloc(GV)) {
  buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
  MI.eraseFromParent();
  return true;
}

LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);

MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
  MachinePointerInfo::getGOT(MF),
  MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
  MachineMemOperand::MOInvariant,
  8 /*Size*/, 8 /*Align*/);

buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);

if (Ty.getSizeInBits() == 32) {
  // Truncate if this is a 32-bit constant adrdess.
  auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
  B.buildExtract(DstReg, Load, 0);
} else
  B.buildLoad(DstReg, GOTAddr, *GOTMMO);

MI.eraseFromParent();
return true;
2010}

2012bool AMDGPULegalizerInfo::legalizeLoad(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, GISelChangeObserver &Observer) const {
B.setInstr(MI);
LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
auto Cast = B.buildAddrSpaceCast(ConstPtr, MI.getOperand(1).getReg());
Observer.changingInstr(MI);
MI.getOperand(1).setReg(Cast.getReg(0));
Observer.changedInstr(MI);
return true;
2022}

2024bool AMDGPULegalizerInfo::legalizeFMad(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
assert(Ty.isScalar())((Ty.isScalar()) ? static_cast<void> (0) : __assert_fail
 ("Ty.isScalar()", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2028, __PRETTY_FUNCTION__));

MachineFunction &MF = B.getMF();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

// TODO: Always legal with future ftz flag.
// FIXME: Do we need just output?
if (Ty == LLT::scalar(32) && !MFI->getMode().allFP32Denormals())
  return true;
if (Ty == LLT::scalar(16) && !MFI->getMode().allFP64FP16Denormals())
  return true;

MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
HelperBuilder.setMBB(*MI.getParent());
return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
2045}

2047bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
Register DstReg = MI.getOperand(0).getReg();
Register PtrReg = MI.getOperand(1).getReg();
Register CmpVal = MI.getOperand(2).getReg();
Register NewVal = MI.getOperand(3).getReg();

assert(SITargetLowering::isFlatGlobalAddrSpace(((SITargetLowering::isFlatGlobalAddrSpace( MRI.getType(PtrReg
).getAddressSpace()) && "this should not have been custom lowered"
) ? static_cast<void> (0) : __assert_fail ("SITargetLowering::isFlatGlobalAddrSpace( MRI.getType(PtrReg).getAddressSpace()) && \"this should not have been custom lowered\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2056, __PRETTY_FUNCTION__))
         MRI.getType(PtrReg).getAddressSpace()) &&((SITargetLowering::isFlatGlobalAddrSpace( MRI.getType(PtrReg
).getAddressSpace()) && "this should not have been custom lowered"
) ? static_cast<void> (0) : __assert_fail ("SITargetLowering::isFlatGlobalAddrSpace( MRI.getType(PtrReg).getAddressSpace()) && \"this should not have been custom lowered\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2056, __PRETTY_FUNCTION__))
       "this should not have been custom lowered")((SITargetLowering::isFlatGlobalAddrSpace( MRI.getType(PtrReg
).getAddressSpace()) && "this should not have been custom lowered"
) ? static_cast<void> (0) : __assert_fail ("SITargetLowering::isFlatGlobalAddrSpace( MRI.getType(PtrReg).getAddressSpace()) && \"this should not have been custom lowered\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2056, __PRETTY_FUNCTION__));

LLT ValTy = MRI.getType(CmpVal);
LLT VecTy = LLT::vector(2, ValTy);

B.setInstr(MI);
Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);

B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
  .addDef(DstReg)
  .addUse(PtrReg)
  .addUse(PackedVal)
  .setMemRefs(MI.memoperands());

MI.eraseFromParent();
return true;
2072}

2074bool AMDGPULegalizerInfo::legalizeFlog(
MachineInstr &MI, MachineIRBuilder &B, double Log2BaseInverted) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT Ty = B.getMRI()->getType(Dst);
unsigned Flags = MI.getFlags();
B.setInstr(MI);

auto Log2Operand = B.buildFLog2(Ty, Src, Flags);
auto Log2BaseInvertedOperand = B.buildFConstant(Ty, Log2BaseInverted);

B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
MI.eraseFromParent();
return true;
2088}

2090bool AMDGPULegalizerInfo::legalizeFExp(MachineInstr &MI,
                                     MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
unsigned Flags = MI.getFlags();
LLT Ty = B.getMRI()->getType(Dst);
B.setInstr(MI);

auto K = B.buildFConstant(Ty, numbers::log2e);
auto Mul = B.buildFMul(Ty, Src, K, Flags);
B.buildFExp2(Dst, Mul, Flags);
MI.eraseFromParent();
return true;
2103}

2105bool AMDGPULegalizerInfo::legalizeFPow(MachineInstr &MI,
                                     MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
Register Src0 = MI.getOperand(1).getReg();
Register Src1 = MI.getOperand(2).getReg();
unsigned Flags = MI.getFlags();
LLT Ty = B.getMRI()->getType(Dst);
B.setInstr(MI);
const LLT S16 = LLT::scalar(16);
const LLT S32 = LLT::scalar(32);

if (Ty == S32) {
  auto Log = B.buildFLog2(S32, Src0, Flags);
  auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {S32}, false)
    .addUse(Log.getReg(0))
    .addUse(Src1)
    .setMIFlags(Flags);
  B.buildFExp2(Dst, Mul, Flags);
} else if (Ty == S16) {
  // There's no f16 fmul_legacy, so we need to convert for it.
  auto Log = B.buildFLog2(S16, Src0, Flags);
  auto Ext0 = B.buildFPExt(S32, Log, Flags);
  auto Ext1 = B.buildFPExt(S32, Src1, Flags);
  auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {S32}, false)
    .addUse(Ext0.getReg(0))
    .addUse(Ext1.getReg(0))
    .setMIFlags(Flags);

  B.buildFExp2(Dst, B.buildFPTrunc(S16, Mul), Flags);
} else
  return false;

MI.eraseFromParent();
return true;
2139}

2141// Find a source register, ignoring any possible source modifiers.
2142static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI) {
Register ModSrc = OrigSrc;
if (MachineInstr *SrcFNeg = getOpcodeDef(AMDGPU::G_FNEG, ModSrc, MRI)) {
  ModSrc = SrcFNeg->getOperand(1).getReg();
  if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
    ModSrc = SrcFAbs->getOperand(1).getReg();
} else if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
  ModSrc = SrcFAbs->getOperand(1).getReg();
return ModSrc;
2151}

2153bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
B.setInstr(MI);

const LLT S1 = LLT::scalar(1);
const LLT S64 = LLT::scalar(64);
Register Dst = MI.getOperand(0).getReg();
Register OrigSrc = MI.getOperand(1).getReg();
unsigned Flags = MI.getFlags();
assert(ST.hasFractBug() && MRI.getType(Dst) == S64 &&((ST.hasFractBug() && MRI.getType(Dst) == S64 &&
 "this should not have been custom lowered") ? static_cast<
void> (0) : __assert_fail ("ST.hasFractBug() && MRI.getType(Dst) == S64 && \"this should not have been custom lowered\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2164, __PRETTY_FUNCTION__))
       "this should not have been custom lowered")((ST.hasFractBug() && MRI.getType(Dst) == S64 &&
 "this should not have been custom lowered") ? static_cast<
void> (0) : __assert_fail ("ST.hasFractBug() && MRI.getType(Dst) == S64 && \"this should not have been custom lowered\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2164, __PRETTY_FUNCTION__));

// V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
// is used instead. However, SI doesn't have V_FLOOR_F64, so the most
// efficient way to implement it is using V_FRACT_F64. The workaround for the
// V_FRACT bug is:
//    fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
//
// Convert floor(x) to (x - fract(x))

auto Fract = B.buildIntrinsic(Intrinsic::amdgcn_fract, {S64}, false)
  .addUse(OrigSrc)
  .setMIFlags(Flags);

// Give source modifier matching some assistance before obscuring a foldable
// pattern.

// TODO: We can avoid the neg on the fract? The input sign to fract
// shouldn't matter?
Register ModSrc = stripAnySourceMods(OrigSrc, MRI);

auto Const = B.buildFConstant(S64, BitsToDouble(0x3fefffffffffffff));

Register Min = MRI.createGenericVirtualRegister(S64);

// We don't need to concern ourselves with the snan handling difference, so
// use the one which will directly select.
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
if (MFI->getMode().IEEE)
  B.buildFMinNumIEEE(Min, Fract, Const, Flags);
else
  B.buildFMinNum(Min, Fract, Const, Flags);

Register CorrectedFract = Min;
if (!MI.getFlag(MachineInstr::FmNoNans)) {
  auto IsNan = B.buildFCmp(CmpInst::FCMP_ORD, S1, ModSrc, ModSrc, Flags);
  CorrectedFract = B.buildSelect(S64, IsNan, ModSrc, Min, Flags).getReg(0);
}

auto NegFract = B.buildFNeg(S64, CorrectedFract, Flags);
B.buildFAdd(Dst, OrigSrc, NegFract, Flags);

MI.eraseFromParent();
return true;
2208}

2210// Turn an illegal packed v2s16 build vector into bit operations.
2211// TODO: This should probably be a bitcast action in LegalizerHelper.
2212bool AMDGPULegalizerInfo::legalizeBuildVector(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
const LLT S32 = LLT::scalar(32);
const LLT V2S16 = LLT::vector(2, 16);
(void)DstTy;
(void)V2S16;
assert(DstTy == V2S16)((DstTy == V2S16) ? static_cast<void> (0) : __assert_fail
 ("DstTy == V2S16", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2220, __PRETTY_FUNCTION__));

Register Src0 = MI.getOperand(1).getReg();
Register Src1 = MI.getOperand(2).getReg();
assert(MRI.getType(Src0) == LLT::scalar(16))((MRI.getType(Src0) == LLT::scalar(16)) ? static_cast<void
> (0) : __assert_fail ("MRI.getType(Src0) == LLT::scalar(16)"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2224, __PRETTY_FUNCTION__));

B.setInstr(MI);
auto Merge = B.buildMerge(S32, {Src0, Src1});
B.buildBitcast(Dst, Merge);

MI.eraseFromParent();
return true;
2232}

2234// Return the use branch instruction, otherwise null if the usage is invalid.
2235static MachineInstr *verifyCFIntrinsic(MachineInstr &MI,
                                     MachineRegisterInfo &MRI,
                                     MachineInstr *&Br) {
Register CondDef = MI.getOperand(0).getReg();
if (!MRI.hasOneNonDBGUse(CondDef))
  return nullptr;

MachineInstr &UseMI = *MRI.use_instr_nodbg_begin(CondDef);
if (UseMI.getParent() != MI.getParent() ||
    UseMI.getOpcode() != AMDGPU::G_BRCOND)
  return nullptr;

// Make sure the cond br is followed by a G_BR
MachineBasicBlock::iterator Next = std::next(UseMI.getIterator());
if (Next != MI.getParent()->end()) {
  if (Next->getOpcode() != AMDGPU::G_BR)
    return nullptr;
  Br = &*Next;
}

return &UseMI;
2256}

2258Register AMDGPULegalizerInfo::getLiveInRegister(MachineRegisterInfo &MRI,
                                              Register Reg, LLT Ty) const {
Register LiveIn = MRI.getLiveInVirtReg(Reg);
if (LiveIn)
  return LiveIn;

Register NewReg = MRI.createGenericVirtualRegister(Ty);
MRI.addLiveIn(Reg, NewReg);
return NewReg;
2267}

2269bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
                                       const ArgDescriptor *Arg) const {
if (!Arg->isRegister() || !Arg->getRegister().isValid())
9
←
Taking false branch→
  return false; // TODO: Handle these

assert(Arg->getRegister().isPhysical())((Arg->getRegister().isPhysical()) ? static_cast<void>
 (0) : __assert_fail ("Arg->getRegister().isPhysical()", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2274, __PRETTY_FUNCTION__));
10
←
'?' condition is true→

MachineRegisterInfo &MRI = *B.getMRI();

LLT Ty = MRI.getType(DstReg);
Register LiveIn = getLiveInRegister(MRI, Arg->getRegister(), Ty);

if (Arg->isMasked()) {
11
←
Calling 'ArgDescriptor::isMasked'→
14
←
Returning from 'ArgDescriptor::isMasked'→
15
←
Taking true branch→
  // TODO: Should we try to emit this once in the entry block?
  const LLT S32 = LLT::scalar(32);
  const unsigned Mask = Arg->getMask();
  const unsigned Shift = countTrailingZeros<unsigned>(Mask);
16
←
Calling 'countTrailingZeros<unsigned int>'→
23
←
Returning from 'countTrailingZeros<unsigned int>'→
24
←
'Shift' initialized to 32→

  Register AndMaskSrc = LiveIn;

  if (Shift24.1
'Shift' is not equal to 0
1
'Shift' is not equal to 0
1
'Shift' is not equal to 0
 != 0) {
25
←
Taking true branch→
    auto ShiftAmt = B.buildConstant(S32, Shift);
    AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0);
  }

  B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(S32, Mask >> Shift));
26
←
The result of the right shift is undefined due to shifting by '32', which is greater or equal to the width of type 'unsigned int'
} else
  B.buildCopy(DstReg, LiveIn);

// Insert the argument copy if it doens't already exist.
// FIXME: It seems EmitLiveInCopies isn't called anywhere?
if (!MRI.getVRegDef(LiveIn)) {
  // FIXME: Should have scoped insert pt
  MachineBasicBlock &OrigInsBB = B.getMBB();
  auto OrigInsPt = B.getInsertPt();

  MachineBasicBlock &EntryMBB = B.getMF().front();
  EntryMBB.addLiveIn(Arg->getRegister());
  B.setInsertPt(EntryMBB, EntryMBB.begin());
  B.buildCopy(LiveIn, Arg->getRegister());

  B.setInsertPt(OrigInsBB, OrigInsPt);
}

return true;
2314}

2316bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B,
AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
B.setInstr(MI);

const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();

const ArgDescriptor *Arg;
const TargetRegisterClass *RC;
std::tie(Arg, RC) = MFI->getPreloadedValue(ArgType);
if (!Arg) {
6
←
Assuming 'Arg' is non-null→
7
←
Taking false branch→
  LLVM_DEBUG(dbgs() << "Required arg register missing\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("amdgpu-legalinfo")) { dbgs() << "Required arg register missing\n"
; } } while (false);
  return false;
}

if (loadInputValue(MI.getOperand(0).getReg(), B, Arg)) {
8
←
Calling 'AMDGPULegalizerInfo::loadInputValue'→
  MI.eraseFromParent();
  return true;
}

return false;
2339}

2341bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
                                     MachineRegisterInfo &MRI,
                                     MachineIRBuilder &B) const {
B.setInstr(MI);
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
LLT S16 = LLT::scalar(16);
LLT S32 = LLT::scalar(32);
LLT S64 = LLT::scalar(64);

if (legalizeFastUnsafeFDIV(MI, MRI, B))
  return true;

if (DstTy == S16)
  return legalizeFDIV16(MI, MRI, B);
if (DstTy == S32)
  return legalizeFDIV32(MI, MRI, B);
if (DstTy == S64)
  return legalizeFDIV64(MI, MRI, B);

return false;
2362}

2364static Register buildDivRCP(MachineIRBuilder &B, Register Src) {
const LLT S32 = LLT::scalar(32);

auto Cvt0 = B.buildUITOFP(S32, Src);
auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {S32}, {Cvt0});
auto FPUIntMaxPlus1 = B.buildFConstant(S32, BitsToFloat(0x4f800000));
auto Mul = B.buildFMul(S32, RcpIFlag, FPUIntMaxPlus1);
return B.buildFPTOUI(S32, Mul).getReg(0);
2372}

2374void AMDGPULegalizerInfo::legalizeUDIV_UREM32Impl(MachineIRBuilder &B,
                                                Register DstReg,
                                                Register Num,
                                                Register Den,
                                                bool IsRem) const {
const LLT S1 = LLT::scalar(1);
const LLT S32 = LLT::scalar(32);

// RCP =  URECIP(Den) = 2^32 / Den + e
// e is rounding error.
auto RCP = buildDivRCP(B, Den);

// RCP_LO = mul(RCP, Den)
auto RCP_LO = B.buildMul(S32, RCP, Den);

// RCP_HI = mulhu (RCP, Den) */
auto RCP_HI = B.buildUMulH(S32, RCP, Den);

// NEG_RCP_LO = -RCP_LO
auto Zero = B.buildConstant(S32, 0);
auto NEG_RCP_LO = B.buildSub(S32, Zero, RCP_LO);

// ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
auto CmpRcpHiZero = B.buildICmp(CmpInst::ICMP_EQ, S1, RCP_HI, Zero);
auto ABS_RCP_LO = B.buildSelect(S32, CmpRcpHiZero, NEG_RCP_LO, RCP_LO);

// Calculate the rounding error from the URECIP instruction
// E = mulhu(ABS_RCP_LO, RCP)
auto E = B.buildUMulH(S32, ABS_RCP_LO, RCP);

// RCP_A_E = RCP + E
auto RCP_A_E = B.buildAdd(S32, RCP, E);

// RCP_S_E = RCP - E
auto RCP_S_E = B.buildSub(S32, RCP, E);

// Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
auto Tmp0 = B.buildSelect(S32, CmpRcpHiZero, RCP_A_E, RCP_S_E);

// Quotient = mulhu(Tmp0, Num)stmp
auto Quotient = B.buildUMulH(S32, Tmp0, Num);

// Num_S_Remainder = Quotient * Den
auto Num_S_Remainder = B.buildMul(S32, Quotient, Den);

// Remainder = Num - Num_S_Remainder
auto Remainder = B.buildSub(S32, Num, Num_S_Remainder);

// Remainder_GE_Den = Remainder >= Den
auto Remainder_GE_Den = B.buildICmp(CmpInst::ICMP_UGE, S1, Remainder, Den);

// Remainder_GE_Zero = Num >= Num_S_Remainder;
auto Remainder_GE_Zero = B.buildICmp(CmpInst::ICMP_UGE, S1,
                                     Num, Num_S_Remainder);

// Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
auto Tmp1 = B.buildAnd(S1, Remainder_GE_Den, Remainder_GE_Zero);

// Calculate Division result:

// Quotient_A_One = Quotient + 1
auto One = B.buildConstant(S32, 1);
auto Quotient_A_One = B.buildAdd(S32, Quotient, One);

// Quotient_S_One = Quotient - 1
auto Quotient_S_One = B.buildSub(S32, Quotient, One);

// Div = (Tmp1 == 0 ? Quotient_A_One : Quotient)
auto Div = B.buildSelect(S32, Tmp1, Quotient, Quotient_A_One);

// Div = (Remainder_GE_Zero ? Div : Quotient_S_One)
if (IsRem) {
  Div = B.buildSelect(S32, Remainder_GE_Zero, Div, Quotient_S_One);

  // Calculate Rem result:
  auto Remainder_S_Den = B.buildSub(S32, Remainder, Den);

  // Remainder_A_Den = Remainder + Den
  auto Remainder_A_Den = B.buildAdd(S32, Remainder, Den);

  // Rem = (Tmp1 ? Remainder_S_Den : Remainder)
  auto Rem = B.buildSelect(S32, Tmp1, Remainder_S_Den, Remainder);

  // Rem = (Remainder_GE_Zero ? Rem : Remainder_A_Den)
  B.buildSelect(DstReg, Remainder_GE_Zero, Rem, Remainder_A_Den);
} else {
  B.buildSelect(DstReg, Remainder_GE_Zero, Div, Quotient_S_One);
}
2462}

2464bool AMDGPULegalizerInfo::legalizeUDIV_UREM32(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B) const {
B.setInstr(MI);
const bool IsRem = MI.getOpcode() == AMDGPU::G_UREM;
Register DstReg = MI.getOperand(0).getReg();
Register Num = MI.getOperand(1).getReg();
Register Den = MI.getOperand(2).getReg();
legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsRem);
MI.eraseFromParent();
return true;
2475}

2477bool AMDGPULegalizerInfo::legalizeUDIV_UREM(MachineInstr &MI,
                                          MachineRegisterInfo &MRI,
                                          MachineIRBuilder &B) const {
if (MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32))
  return legalizeUDIV_UREM32(MI, MRI, B);
return false;
2483}

2485bool AMDGPULegalizerInfo::legalizeSDIV_SREM32(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B) const {
B.setInstr(MI);
const LLT S32 = LLT::scalar(32);

const bool IsRem = MI.getOpcode() == AMDGPU::G_SREM;
Register DstReg = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();

auto ThirtyOne = B.buildConstant(S32, 31);
auto LHSign = B.buildAShr(S32, LHS, ThirtyOne);
auto RHSign = B.buildAShr(S32, LHS, ThirtyOne);

LHS = B.buildAdd(S32, LHS, LHSign).getReg(0);
RHS = B.buildAdd(S32, RHS, RHSign).getReg(0);

LHS = B.buildXor(S32, LHS, LHSign).getReg(0);
RHS = B.buildXor(S32, RHS, RHSign).getReg(0);

Register UDivRem = MRI.createGenericVirtualRegister(S32);
legalizeUDIV_UREM32Impl(B, UDivRem, LHS, RHS, IsRem);

if (IsRem) {
  auto RSign = LHSign; // Remainder sign is the same as LHS
  UDivRem = B.buildXor(S32, UDivRem, RSign).getReg(0);
  B.buildSub(DstReg, UDivRem, RSign);
} else {
  auto DSign = B.buildXor(S32, LHSign, RHSign);
  UDivRem = B.buildXor(S32, UDivRem, DSign).getReg(0);
  B.buildSub(DstReg, UDivRem, DSign);
}

MI.eraseFromParent();
return true;
2521}

2523bool AMDGPULegalizerInfo::legalizeSDIV_SREM(MachineInstr &MI,
                                          MachineRegisterInfo &MRI,
                                          MachineIRBuilder &B) const {
if (MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32))
  return legalizeSDIV_SREM32(MI, MRI, B);
return false;
2529}

2531bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
                                               MachineRegisterInfo &MRI,
                                               MachineIRBuilder &B) const {
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();

uint16_t Flags = MI.getFlags();

LLT ResTy = MRI.getType(Res);
LLT S32 = LLT::scalar(32);
LLT S64 = LLT::scalar(64);

const MachineFunction &MF = B.getMF();
bool Unsafe =
  MF.getTarget().Options.UnsafeFPMath || MI.getFlag(MachineInstr::FmArcp);

if (!MF.getTarget().Options.UnsafeFPMath && ResTy == S64)
  return false;

if (!Unsafe && ResTy == S32 &&
    MF.getInfo<SIMachineFunctionInfo>()->getMode().allFP32Denormals())
  return false;

if (auto CLHS = getConstantFPVRegVal(LHS, MRI)) {
  // 1 / x -> RCP(x)
  if (CLHS->isExactlyValue(1.0)) {
    B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res, false)
      .addUse(RHS)
      .setMIFlags(Flags);

    MI.eraseFromParent();
    return true;
  }

  // -1 / x -> RCP( FNEG(x) )
  if (CLHS->isExactlyValue(-1.0)) {
    auto FNeg = B.buildFNeg(ResTy, RHS, Flags);
    B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res, false)
      .addUse(FNeg.getReg(0))
      .setMIFlags(Flags);

    MI.eraseFromParent();
    return true;
  }
}

// x / y -> x * (1.0 / y)
if (Unsafe) {
  auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy}, false)
    .addUse(RHS)
    .setMIFlags(Flags);
  B.buildFMul(Res, LHS, RCP, Flags);

  MI.eraseFromParent();
  return true;
}

return false;
2590}

2592bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
B.setInstr(MI);
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();

uint16_t Flags = MI.getFlags();

LLT S16 = LLT::scalar(16);
LLT S32 = LLT::scalar(32);

auto LHSExt = B.buildFPExt(S32, LHS, Flags);
auto RHSExt = B.buildFPExt(S32, RHS, Flags);

auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false)
  .addUse(RHSExt.getReg(0))
  .setMIFlags(Flags);

auto QUOT = B.buildFMul(S32, LHSExt, RCP, Flags);
auto RDst = B.buildFPTrunc(S16, QUOT, Flags);

B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res, false)
  .addUse(RDst.getReg(0))
  .addUse(RHS)
  .addUse(LHS)
  .setMIFlags(Flags);

MI.eraseFromParent();
return true;
2623}

2625// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
2626// to enable denorm mode. When 'Enable' is false, disable denorm mode.
2627static void toggleSPDenormMode(bool Enable,
                             MachineIRBuilder &B,
                             const GCNSubtarget &ST,
                             AMDGPU::SIModeRegisterDefaults Mode) {
// Set SP denorm mode to this value.
unsigned SPDenormMode =
  Enable ? FP_DENORM_FLUSH_NONE3 : Mode.fpDenormModeSPValue();

if (ST.hasDenormModeInst()) {
  // Preserve default FP64FP16 denorm mode while updating FP32 mode.
  uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();

  uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
  B.buildInstr(AMDGPU::S_DENORM_MODE)
    .addImm(NewDenormModeValue);

} else {
  // Select FP32 bit field in mode register.
  unsigned SPDenormModeBitField = AMDGPU::Hwreg::ID_MODE |
                                  (4 << AMDGPU::Hwreg::OFFSET_SHIFT_) |
                                  (1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_);

  B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
    .addImm(SPDenormMode)
    .addImm(SPDenormModeBitField);
}
2653}

2655bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
B.setInstr(MI);
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
AMDGPU::SIModeRegisterDefaults Mode = MFI->getMode();

uint16_t Flags = MI.getFlags();

LLT S32 = LLT::scalar(32);
LLT S1 = LLT::scalar(1);

auto One = B.buildFConstant(S32, 1.0f);

auto DenominatorScaled =
  B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}, false)
    .addUse(RHS)
    .addUse(LHS)
    .addImm(1)
    .setMIFlags(Flags);
auto NumeratorScaled =
  B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}, false)
    .addUse(LHS)
    .addUse(RHS)
    .addImm(0)
    .setMIFlags(Flags);

auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false)
  .addUse(DenominatorScaled.getReg(0))
  .setMIFlags(Flags);
auto NegDivScale0 = B.buildFNeg(S32, DenominatorScaled, Flags);

// FIXME: Doesn't correctly model the FP mode switch, and the FP operations
// aren't modeled as reading it.
if (!Mode.allFP32Denormals())
  toggleSPDenormMode(true, B, ST, Mode);

auto Fma0 = B.buildFMA(S32, NegDivScale0, ApproxRcp, One, Flags);
auto Fma1 = B.buildFMA(S32, Fma0, ApproxRcp, ApproxRcp, Flags);
auto Mul = B.buildFMul(S32, NumeratorScaled, Fma1, Flags);
auto Fma2 = B.buildFMA(S32, NegDivScale0, Mul, NumeratorScaled, Flags);
auto Fma3 = B.buildFMA(S32, Fma2, Fma1, Mul, Flags);
auto Fma4 = B.buildFMA(S32, NegDivScale0, Fma3, NumeratorScaled, Flags);

if (!Mode.allFP32Denormals())
  toggleSPDenormMode(false, B, ST, Mode);

auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S32}, false)
  .addUse(Fma4.getReg(0))
  .addUse(Fma1.getReg(0))
  .addUse(Fma3.getReg(0))
  .addUse(NumeratorScaled.getReg(1))
  .setMIFlags(Flags);

B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res, false)
  .addUse(Fmas.getReg(0))
  .addUse(RHS)
  .addUse(LHS)
  .setMIFlags(Flags);

MI.eraseFromParent();
return true;
2720}

2722bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
B.setInstr(MI);
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();

uint16_t Flags = MI.getFlags();

LLT S64 = LLT::scalar(64);
LLT S1 = LLT::scalar(1);

auto One = B.buildFConstant(S64, 1.0);

auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1}, false)
  .addUse(LHS)
  .addUse(RHS)
  .addImm(1)
  .setMIFlags(Flags);

auto NegDivScale0 = B.buildFNeg(S64, DivScale0.getReg(0), Flags);

auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S64}, false)
  .addUse(DivScale0.getReg(0))
  .setMIFlags(Flags);

auto Fma0 = B.buildFMA(S64, NegDivScale0, Rcp, One, Flags);
auto Fma1 = B.buildFMA(S64, Rcp, Fma0, Rcp, Flags);
auto Fma2 = B.buildFMA(S64, NegDivScale0, Fma1, One, Flags);

auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1}, false)
  .addUse(LHS)
  .addUse(RHS)
  .addImm(0)
  .setMIFlags(Flags);

auto Fma3 = B.buildFMA(S64, Fma1, Fma2, Fma1, Flags);
auto Mul = B.buildMul(S64, DivScale1.getReg(0), Fma3, Flags);
auto Fma4 = B.buildFMA(S64, NegDivScale0, Mul, DivScale1.getReg(0), Flags);

Register Scale;
if (!ST.hasUsableDivScaleConditionOutput()) {
  // Workaround a hardware bug on SI where the condition output from div_scale
  // is not usable.

  LLT S32 = LLT::scalar(32);

  auto NumUnmerge = B.buildUnmerge(S32, LHS);
  auto DenUnmerge = B.buildUnmerge(S32, RHS);
  auto Scale0Unmerge = B.buildUnmerge(S32, DivScale0);
  auto Scale1Unmerge = B.buildUnmerge(S32, DivScale1);

  auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1),
                            Scale1Unmerge.getReg(1));
  auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1),
                            Scale0Unmerge.getReg(1));
  Scale = B.buildXor(S1, CmpNum, CmpDen).getReg(0);
} else {
  Scale = DivScale1.getReg(1);
}

auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S64}, false)
  .addUse(Fma4.getReg(0))
  .addUse(Fma3.getReg(0))
  .addUse(Mul.getReg(0))
  .addUse(Scale)
  .setMIFlags(Flags);

B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, makeArrayRef(Res), false)
  .addUse(Fmas.getReg(0))
  .addUse(RHS)
  .addUse(LHS)
  .setMIFlags(Flags);

MI.eraseFromParent();
return true;
2799}

2801bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
                                               MachineRegisterInfo &MRI,
                                               MachineIRBuilder &B) const {
B.setInstr(MI);
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(2).getReg();
Register RHS = MI.getOperand(3).getReg();
uint16_t Flags = MI.getFlags();

LLT S32 = LLT::scalar(32);
LLT S1 = LLT::scalar(1);

auto Abs = B.buildFAbs(S32, RHS, Flags);
const APFloat C0Val(1.0f);

auto C0 = B.buildConstant(S32, 0x6f800000);
auto C1 = B.buildConstant(S32, 0x2f800000);
auto C2 = B.buildConstant(S32, FloatToBits(1.0f));

auto CmpRes = B.buildFCmp(CmpInst::FCMP_OGT, S1, Abs, C0, Flags);
auto Sel = B.buildSelect(S32, CmpRes, C1, C2, Flags);

auto Mul0 = B.buildFMul(S32, RHS, Sel, Flags);

auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false)
  .addUse(Mul0.getReg(0))
  .setMIFlags(Flags);

auto Mul1 = B.buildFMul(S32, LHS, RCP, Flags);

B.buildFMul(Res, Sel, Mul1, Flags);

MI.eraseFromParent();
return true;
2835}

2837bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
                                               MachineRegisterInfo &MRI,
                                               MachineIRBuilder &B) const {
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
if (!MFI->isEntryFunction()) {
3
←
Assuming the condition is true→
4
←
Taking true branch→
  return legalizePreloadedArgIntrin(MI, MRI, B,
5
←
Calling 'AMDGPULegalizerInfo::legalizePreloadedArgIntrin'→
                                    AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
}

B.setInstr(MI);

uint64_t Offset =
  ST.getTargetLowering()->getImplicitParameterOffset(
    B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
Register DstReg = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(DstReg);
LLT IdxTy = LLT::scalar(DstTy.getSizeInBits());

const ArgDescriptor *Arg;
const TargetRegisterClass *RC;
std::tie(Arg, RC)
  = MFI->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
if (!Arg)
  return false;

Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
if (!loadInputValue(KernargPtrReg, B, Arg))
  return false;

B.buildPtrAdd(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0));
MI.eraseFromParent();
return true;
2869}

2871bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B,
                                            unsigned AddrSpace) const {
B.setInstr(MI);
Register ApertureReg = getSegmentAperture(AddrSpace, MRI, B);
auto Hi32 = B.buildExtract(LLT::scalar(32), MI.getOperand(2).getReg(), 32);
B.buildICmp(ICmpInst::ICMP_EQ, MI.getOperand(0), Hi32, ApertureReg);
MI.eraseFromParent();
return true;
2881}

2883// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
2884// offset (the offset that is included in bounds checking and swizzling, to be
2885// split between the instruction's voffset and immoffset fields) and soffset
2886// (the offset that is excluded from bounds checking and swizzling, to go in
2887// the instruction's soffset field).  This function takes the first kind of
2888// offset and figures out how to split it between voffset and immoffset.
2889std::tuple<Register, unsigned, unsigned>
2890AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
                                      Register OrigOffset) const {
const unsigned MaxImm = 4095;
Register BaseReg;
unsigned TotalConstOffset;
MachineInstr *OffsetDef;
const LLT S32 = LLT::scalar(32);

std::tie(BaseReg, TotalConstOffset, OffsetDef)
  = AMDGPU::getBaseWithConstantOffset(*B.getMRI(), OrigOffset);

unsigned ImmOffset = TotalConstOffset;

// If the immediate value is too big for the immoffset field, put the value
// and -4096 into the immoffset field so that the value that is copied/added
// for the voffset field is a multiple of 4096, and it stands more chance
// of being CSEd with the copy/add for another similar load/store.
// However, do not do that rounding down to a multiple of 4096 if that is a
// negative number, as it appears to be illegal to have a negative offset
// in the vgpr, even if adding the immediate offset makes it positive.
unsigned Overflow = ImmOffset & ~MaxImm;
ImmOffset -= Overflow;
if ((int32_t)Overflow < 0) {
  Overflow += ImmOffset;
  ImmOffset = 0;
}

if (Overflow != 0) {
  if (!BaseReg) {
    BaseReg = B.buildConstant(S32, Overflow).getReg(0);
  } else {
    auto OverflowVal = B.buildConstant(S32, Overflow);
    BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
  }
}

if (!BaseReg)
  BaseReg = B.buildConstant(S32, 0).getReg(0);

return std::make_tuple(BaseReg, ImmOffset, TotalConstOffset);
2930}

2932/// Handle register layout difference for f16 images for some subtargets.
2933Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
                                           MachineRegisterInfo &MRI,
                                           Register Reg) const {
if (!ST.hasUnpackedD16VMem())
  return Reg;

const LLT S16 = LLT::scalar(16);
const LLT S32 = LLT::scalar(32);
LLT StoreVT = MRI.getType(Reg);
assert(StoreVT.isVector() && StoreVT.getElementType() == S16)((StoreVT.isVector() && StoreVT.getElementType() == S16
) ? static_cast<void> (0) : __assert_fail ("StoreVT.isVector() && StoreVT.getElementType() == S16"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 2942, __PRETTY_FUNCTION__));

auto Unmerge = B.buildUnmerge(S16, Reg);

SmallVector<Register, 4> WideRegs;
for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
  WideRegs.push_back(B.buildAnyExt(S32, Unmerge.getReg(I)).getReg(0));

int NumElts = StoreVT.getNumElements();

return B.buildBuildVector(LLT::vector(NumElts, S32), WideRegs).getReg(0);
2953}

2955Register AMDGPULegalizerInfo::fixStoreSourceType(
MachineIRBuilder &B, Register VData, bool IsFormat) const {
MachineRegisterInfo *MRI = B.getMRI();
LLT Ty = MRI->getType(VData);

const LLT S16 = LLT::scalar(16);

// Fixup illegal register types for i8 stores.
if (Ty == LLT::scalar(8) || Ty == S16) {
  Register AnyExt = B.buildAnyExt(LLT::scalar(32), VData).getReg(0);
  return AnyExt;
}

if (Ty.isVector()) {
  if (Ty.getElementType() == S16 && Ty.getNumElements() <= 4) {
    if (IsFormat)
      return handleD16VData(B, *MRI, VData);
  }
}

return VData;
2976}

2978bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B,
                                            bool IsTyped,
                                            bool IsFormat) const {
B.setInstr(MI);

Register VData = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(VData);
LLT EltTy = Ty.getScalarType();
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const LLT S32 = LLT::scalar(32);

VData = fixStoreSourceType(B, VData, IsFormat);
Register RSrc = MI.getOperand(2).getReg();

MachineMemOperand *MMO = *MI.memoperands_begin();
const int MemSize = MMO->getSize();

unsigned ImmOffset;
unsigned TotalOffset;

// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;

// The struct intrinsic variants add one additional operand over raw.
const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
Register VIndex;
int OpOffset = 0;
if (HasVIndex) {
  VIndex = MI.getOperand(3).getReg();
  OpOffset = 1;
}

Register VOffset = MI.getOperand(3 + OpOffset).getReg();
Register SOffset = MI.getOperand(4 + OpOffset).getReg();

unsigned Format = 0;
if (IsTyped) {
  Format = MI.getOperand(5 + OpOffset).getImm();
  ++OpOffset;
}

unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();

std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
if (TotalOffset != 0)
  MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MemSize);

unsigned Opc;
if (IsTyped) {
  Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
                AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
} else if (IsFormat) {
  Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
                AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
} else {
  switch (MemSize) {
  case 1:
    Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
    break;
  case 2:
    Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
    break;
  default:
    Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
    break;
  }
}

if (!VIndex)
  VIndex = B.buildConstant(S32, 0).getReg(0);

auto MIB = B.buildInstr(Opc)
  .addUse(VData)              // vdata
  .addUse(RSrc)               // rsrc
  .addUse(VIndex)             // vindex
  .addUse(VOffset)            // voffset
  .addUse(SOffset)            // soffset
  .addImm(ImmOffset);         // offset(imm)

if (IsTyped)
  MIB.addImm(Format);

MIB.addImm(AuxiliaryData)      // cachepolicy, swizzled buffer(imm)
   .addImm(HasVIndex ? -1 : 0) // idxen(imm)
   .addMemOperand(MMO);

MI.eraseFromParent();
return true;
3068}

3070bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           MachineIRBuilder &B,
                                           bool IsFormat,
                                           bool IsTyped) const {
B.setInstr(MI);

// FIXME: Verifier should enforce 1 MMO for these intrinsics.
MachineMemOperand *MMO = *MI.memoperands_begin();
const int MemSize = MMO->getSize();
const LLT S32 = LLT::scalar(32);

Register Dst = MI.getOperand(0).getReg();
Register RSrc = MI.getOperand(2).getReg();

// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;

// The struct intrinsic variants add one additional operand over raw.
const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
Register VIndex;
int OpOffset = 0;
if (HasVIndex) {
  VIndex = MI.getOperand(3).getReg();
  OpOffset = 1;
}

Register VOffset = MI.getOperand(3 + OpOffset).getReg();
Register SOffset = MI.getOperand(4 + OpOffset).getReg();

unsigned Format = 0;
if (IsTyped) {
  Format = MI.getOperand(5 + OpOffset).getImm();
  ++OpOffset;
}

unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
unsigned ImmOffset;
unsigned TotalOffset;

LLT Ty = MRI.getType(Dst);
LLT EltTy = Ty.getScalarType();
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const bool Unpacked = ST.hasUnpackedD16VMem();

std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
if (TotalOffset != 0)
  MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MemSize);

unsigned Opc;

if (IsTyped) {
  Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
                AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
} else if (IsFormat) {
  Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16 :
                AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
} else {
  switch (MemSize) {
  case 1:
    Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
    break;
  case 2:
    Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
    break;
  default:
    Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
    break;
  }
}

Register LoadDstReg;

bool IsExtLoad = (!IsD16 && MemSize < 4) || (IsD16 && !Ty.isVector());
LLT UnpackedTy = Ty.changeElementSize(32);

if (IsExtLoad)
  LoadDstReg = B.getMRI()->createGenericVirtualRegister(S32);
else if (Unpacked && IsD16 && Ty.isVector())
  LoadDstReg = B.getMRI()->createGenericVirtualRegister(UnpackedTy);
else
  LoadDstReg = Dst;

if (!VIndex)
  VIndex = B.buildConstant(S32, 0).getReg(0);

auto MIB = B.buildInstr(Opc)
  .addDef(LoadDstReg)         // vdata
  .addUse(RSrc)               // rsrc
  .addUse(VIndex)             // vindex
  .addUse(VOffset)            // voffset
  .addUse(SOffset)            // soffset
  .addImm(ImmOffset);         // offset(imm)

if (IsTyped)
  MIB.addImm(Format);

MIB.addImm(AuxiliaryData)      // cachepolicy, swizzled buffer(imm)
   .addImm(HasVIndex ? -1 : 0) // idxen(imm)
   .addMemOperand(MMO);

if (LoadDstReg != Dst) {
  B.setInsertPt(B.getMBB(), ++B.getInsertPt());

  // Widen result for extending loads was widened.
  if (IsExtLoad)
    B.buildTrunc(Dst, LoadDstReg);
  else {
    // Repack to original 16-bit vector result
    // FIXME: G_TRUNC should work, but legalization currently fails
    auto Unmerge = B.buildUnmerge(S32, LoadDstReg);
    SmallVector<Register, 4> Repack;
    for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
      Repack.push_back(B.buildTrunc(EltTy, Unmerge.getReg(I)).getReg(0));
    B.buildMerge(Dst, Repack);
  }
}

MI.eraseFromParent();
return true;
3190}

3192bool AMDGPULegalizerInfo::legalizeAtomicIncDec(MachineInstr &MI,
                                             MachineIRBuilder &B,
                                             bool IsInc) const {
B.setInstr(MI);
unsigned Opc = IsInc ? AMDGPU::G_AMDGPU_ATOMIC_INC :
                       AMDGPU::G_AMDGPU_ATOMIC_DEC;
B.buildInstr(Opc)
  .addDef(MI.getOperand(0).getReg())
  .addUse(MI.getOperand(2).getReg())
  .addUse(MI.getOperand(3).getReg())
  .cloneMemRefs(MI);
MI.eraseFromParent();
return true;
3205}

3207static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
switch (IntrID) {
case Intrinsic::amdgcn_raw_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_buffer_atomic_swap:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
case Intrinsic::amdgcn_raw_buffer_atomic_add:
case Intrinsic::amdgcn_struct_buffer_atomic_add:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
case Intrinsic::amdgcn_raw_buffer_atomic_sub:
case Intrinsic::amdgcn_struct_buffer_atomic_sub:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
case Intrinsic::amdgcn_raw_buffer_atomic_smin:
case Intrinsic::amdgcn_struct_buffer_atomic_smin:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
case Intrinsic::amdgcn_raw_buffer_atomic_umin:
case Intrinsic::amdgcn_struct_buffer_atomic_umin:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
case Intrinsic::amdgcn_raw_buffer_atomic_smax:
case Intrinsic::amdgcn_struct_buffer_atomic_smax:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
case Intrinsic::amdgcn_raw_buffer_atomic_umax:
case Intrinsic::amdgcn_struct_buffer_atomic_umax:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
case Intrinsic::amdgcn_raw_buffer_atomic_and:
case Intrinsic::amdgcn_struct_buffer_atomic_and:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
case Intrinsic::amdgcn_raw_buffer_atomic_or:
case Intrinsic::amdgcn_struct_buffer_atomic_or:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
case Intrinsic::amdgcn_raw_buffer_atomic_xor:
case Intrinsic::amdgcn_struct_buffer_atomic_xor:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
case Intrinsic::amdgcn_raw_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_buffer_atomic_inc:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
case Intrinsic::amdgcn_raw_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_buffer_atomic_dec:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
  return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
default:
  llvm_unreachable("unhandled atomic opcode")::llvm::llvm_unreachable_internal("unhandled atomic opcode", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 3249);
}
3251}

3253bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
                                             MachineIRBuilder &B,
                                             Intrinsic::ID IID) const {
B.setInstr(MI);

const bool IsCmpSwap = IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
                       IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap;

Register Dst = MI.getOperand(0).getReg();
Register VData = MI.getOperand(2).getReg();

Register CmpVal;
int OpOffset = 0;

if (IsCmpSwap) {
  CmpVal = MI.getOperand(3 + OpOffset).getReg();
  ++OpOffset;
}

Register RSrc = MI.getOperand(3 + OpOffset).getReg();
const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;

// The struct intrinsic variants add one additional operand over raw.
const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
Register VIndex;
if (HasVIndex) {
  VIndex = MI.getOperand(4 + OpOffset).getReg();
  ++OpOffset;
}

Register VOffset = MI.getOperand(4 + OpOffset).getReg();
Register SOffset = MI.getOperand(5 + OpOffset).getReg();
unsigned AuxiliaryData = MI.getOperand(6 + OpOffset).getImm();

MachineMemOperand *MMO = *MI.memoperands_begin();

unsigned ImmOffset;
unsigned TotalOffset;
std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
if (TotalOffset != 0)
  MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MMO->getSize());

if (!VIndex)
  VIndex = B.buildConstant(LLT::scalar(32), 0).getReg(0);

auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
  .addDef(Dst)
  .addUse(VData); // vdata

if (IsCmpSwap)
  MIB.addReg(CmpVal);

MIB.addUse(RSrc)               // rsrc
   .addUse(VIndex)             // vindex
   .addUse(VOffset)            // voffset
   .addUse(SOffset)            // soffset
   .addImm(ImmOffset)          // offset(imm)
   .addImm(AuxiliaryData)      // cachepolicy, swizzled buffer(imm)
   .addImm(HasVIndex ? -1 : 0) // idxen(imm)
   .addMemOperand(MMO);

MI.eraseFromParent();
return true;
3316}

3318// Produce a vector of s16 elements from s32 pieces.
3319static void truncToS16Vector(MachineIRBuilder &B, Register DstReg,
                           ArrayRef<Register> UnmergeParts) {
const LLT S16 = LLT::scalar(16);

SmallVector<Register, 4> RemergeParts(UnmergeParts.size());
for (int I = 0, E = UnmergeParts.size(); I != E; ++I)
  RemergeParts[I] = B.buildTrunc(S16, UnmergeParts[I]).getReg(0);

B.buildBuildVector(DstReg, RemergeParts);
3328}

3330/// Convert a set of s32 registers to a result vector with s16 elements.
3331static void bitcastToS16Vector(MachineIRBuilder &B, Register DstReg,
                             ArrayRef<Register> UnmergeParts) {
MachineRegisterInfo &MRI = *B.getMRI();
const LLT V2S16 = LLT::vector(2, 16);
LLT TargetTy = MRI.getType(DstReg);
int NumElts = UnmergeParts.size();

if (NumElts == 1) {
  assert(TargetTy == V2S16)((TargetTy == V2S16) ? static_cast<void> (0) : __assert_fail
 ("TargetTy == V2S16", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 3339, __PRETTY_FUNCTION__));
  B.buildBitcast(DstReg, UnmergeParts[0]);
  return;
}

SmallVector<Register, 4> RemergeParts(NumElts);
for (int I = 0; I != NumElts; ++I)
  RemergeParts[I] = B.buildBitcast(V2S16, UnmergeParts[I]).getReg(0);

if (TargetTy.getSizeInBits() == 32u * NumElts) {
  B.buildConcatVectors(DstReg, RemergeParts);
  return;
}

const LLT V3S16 = LLT::vector(3, 16);
const LLT V6S16 = LLT::vector(6, 16);

// Widen to v6s16 and unpack v3 parts.
assert(TargetTy == V3S16)((TargetTy == V3S16) ? static_cast<void> (0) : __assert_fail
 ("TargetTy == V3S16", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp"
, 3357, __PRETTY_FUNCTION__));

RemergeParts.push_back(B.buildUndef(V2S16).getReg(0));
auto Concat = B.buildConcatVectors(V6S16, RemergeParts);
B.buildUnmerge({DstReg, MRI.createGenericVirtualRegister(V3S16)}, Concat);
3362}

3364// FIXME: Just vector trunc should be sufficent, but legalization currently
3365// broken.
3366static void repackUnpackedD16Load(MachineIRBuilder &B, Register DstReg,
                                Register WideDstReg) {
const LLT S32 = LLT::scalar(32);
const LLT S16 = LLT::scalar(16);

auto Unmerge = B.buildUnmerge(S32, WideDstReg);

int NumOps = Unmerge->getNumOperands() - 1;
SmallVector<Register, 4> RemergeParts(NumOps);
for (int I = 0; I != NumOps; ++I)
  RemergeParts[I] = B.buildTrunc(S16, Unmerge.getReg(I)).getReg(0);

B.buildBuildVector(DstReg, RemergeParts);
3379}

3381bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
  MachineInstr &MI, MachineIRBuilder &B,
  GISelChangeObserver &Observer,
  const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const {
bool IsTFE = MI.getNumExplicitDefs() == 2;

// We are only processing the operands of d16 image operations on subtargets
// that use the unpacked register layout, or need to repack the TFE result.

// TODO: Need to handle a16 images too
// TODO: Do we need to guard against already legalized intrinsics?
if (!IsTFE && !ST.hasUnpackedD16VMem())
  return true;

const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
  AMDGPU::getMIMGBaseOpcodeInfo(ImageDimIntr->BaseOpcode);

if (BaseOpcode->Atomic) // No d16 atomics, or TFE.
  return true;

B.setInstr(MI);

MachineRegisterInfo *MRI = B.getMRI();
const LLT S32 = LLT::scalar(32);
const LLT S16 = LLT::scalar(16);

if (BaseOpcode->Store) { // No TFE for stores?
  Register VData = MI.getOperand(1).getReg();
  LLT Ty = MRI->getType(VData);
  if (!Ty.isVector() || Ty.getElementType() != S16)
    return true;

  B.setInstr(MI);

  Observer.changingInstr(MI);
  MI.getOperand(1).setReg(handleD16VData(B, *MRI, VData));
  Observer.changedInstr(MI);
  return true;
}

Register DstReg = MI.getOperand(0).getReg();
LLT Ty = MRI->getType(DstReg);
const LLT EltTy = Ty.getScalarType();
const bool IsD16 = Ty.getScalarType() == S16;
const unsigned NumElts = Ty.isVector() ? Ty.getNumElements() : 1;

if (IsTFE) {
  // In the IR, TFE is supposed to be used with a 2 element struct return
  // type. The intruction really returns these two values in one contiguous
  // register, with one additional dword beyond the loaded data. Rewrite the
  // return type to use a single register result.
  Register Dst1Reg = MI.getOperand(1).getReg();
  if (MRI->getType(Dst1Reg) != S32)
    return false;

  // TODO: Make sure the TFE operand bit is set.

  // The raw dword aligned data component of the load. The only legal cases
  // where this matters should be when using the packed D16 format, for
  // s16 -> <2 x s16>, and <3 x s16> -> <4 x s16>,
  LLT RoundedTy;
  LLT TFETy;

  if (IsD16 && ST.hasUnpackedD16VMem()) {
    RoundedTy = LLT::scalarOrVector(NumElts, 32);
    TFETy = LLT::vector(NumElts + 1, 32);
  } else {
    unsigned EltSize = Ty.getScalarSizeInBits();
    unsigned RoundedElts = (Ty.getSizeInBits() + 31) / 32;
    unsigned RoundedSize = 32 * RoundedElts;
    RoundedTy = LLT::scalarOrVector(RoundedSize / EltSize, EltSize);
    TFETy = LLT::vector(RoundedSize / 32 + 1, S32);
  }

  Register TFEReg = MRI->createGenericVirtualRegister(TFETy);
  Observer.changingInstr(MI);

  MI.getOperand(0).setReg(TFEReg);
  MI.RemoveOperand(1);

  Observer.changedInstr(MI);

  // Insert after the instruction.
  B.setInsertPt(*MI.getParent(), ++MI.getIterator());

  // Now figure out how to copy the new result register back into the old
  // result.

  SmallVector<Register, 5> UnmergeResults(TFETy.getNumElements(), Dst1Reg);
  int NumDataElts = TFETy.getNumElements() - 1;

  if (!Ty.isVector()) {
    // Simplest case is a trivial unmerge (plus a truncate for d16).
    UnmergeResults[0] = Ty == S32 ?
      DstReg : MRI->createGenericVirtualRegister(S32);

    B.buildUnmerge(UnmergeResults, TFEReg);
    if (Ty != S32)
      B.buildTrunc(DstReg, UnmergeResults[0]);
    return true;
  }

  // We have to repack into a new vector of some kind.
  for (int I = 0; I != NumDataElts; ++I)
    UnmergeResults[I] = MRI->createGenericVirtualRegister(S32);
  B.buildUnmerge(UnmergeResults, TFEReg);

  // Drop the final TFE element.
  ArrayRef<Register> DataPart(UnmergeResults.data(), NumDataElts);

  if (EltTy == S32)
    B.buildBuildVector(DstReg, DataPart);
  else if (ST.hasUnpackedD16VMem())
    truncToS16Vector(B, DstReg, DataPart);
  else
    bitcastToS16Vector(B, DstReg, DataPart);

  return true;
}

// Must be an image load.
if (!Ty.isVector() || Ty.getElementType() != S16)
  return true;

B.setInsertPt(*MI.getParent(), ++MI.getIterator());

LLT WidenedTy = Ty.changeElementType(S32);
Register WideDstReg = MRI->createGenericVirtualRegister(WidenedTy);

Observer.changingInstr(MI);
MI.getOperand(0).setReg(WideDstReg);
Observer.changedInstr(MI);

repackUnpackedD16Load(B, DstReg, WideDstReg);
return true;
3516}

3518bool AMDGPULegalizerInfo::legalizeSBufferLoad(
MachineInstr &MI, MachineIRBuilder &B,
GISelChangeObserver &Observer) const {
Register Dst = MI.getOperand(0).getReg();
LLT Ty = B.getMRI()->getType(Dst);
unsigned Size = Ty.getSizeInBits();
MachineFunction &MF = B.getMF();

Observer.changingInstr(MI);

// FIXME: We don't really need this intermediate instruction. The intrinsic
// should be fixed to have a memory operand. Since it's readnone, we're not
// allowed to add one.
MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_LOAD));
MI.RemoveOperand(1); // Remove intrinsic ID

// FIXME: When intrinsic definition is fixed, this should have an MMO already.
// TODO: Should this use datalayout alignment?
const unsigned MemSize = (Size + 7) / 8;
const unsigned MemAlign = 4;
MachineMemOperand *MMO = MF.getMachineMemOperand(
  MachinePointerInfo(),
  MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
  MachineMemOperand::MOInvariant, MemSize, MemAlign);
MI.addMemOperand(MF, MMO);

// There are no 96-bit result scalar loads, but widening to 128-bit should
// always be legal. We may need to restore this to a 96-bit result if it turns
// out this needs to be converted to a vector load during RegBankSelect.
if (!isPowerOf2_32(Size)) {
  LegalizerHelper Helper(MF, *this, Observer, B);
  B.setInstr(MI);

  if (Ty.isVector())
    Helper.moreElementsVectorDst(MI, getPow2VectorType(Ty), 0);
  else
    Helper.widenScalarDst(MI, getPow2ScalarType(Ty), 0);
}

Observer.changedInstr(MI);
return true;
3559}

3561bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
                                          MachineIRBuilder &B,
                                          GISelChangeObserver &Observer) const {
MachineRegisterInfo &MRI = *B.getMRI();

// Replace the use G_BRCOND with the exec manipulate and branch pseudos.
auto IntrID = MI.getIntrinsicID();
switch (IntrID) {
1
Control jumps to 'case amdgcn_implicitarg_ptr:'  at line 3640→
case Intrinsic::amdgcn_if:
case Intrinsic::amdgcn_else: {
  MachineInstr *Br = nullptr;
  if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI, Br)) {
    const SIRegisterInfo *TRI
      = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());

    B.setInstr(*BrCond);
    Register Def = MI.getOperand(1).getReg();
    Register Use = MI.getOperand(3).getReg();

    MachineBasicBlock *BrTarget = BrCond->getOperand(1).getMBB();
    if (Br)
      BrTarget = Br->getOperand(0).getMBB();

    if (IntrID == Intrinsic::amdgcn_if) {
      B.buildInstr(AMDGPU::SI_IF)
        .addDef(Def)
        .addUse(Use)
        .addMBB(BrTarget);
    } else {
      B.buildInstr(AMDGPU::SI_ELSE)
        .addDef(Def)
        .addUse(Use)
        .addMBB(BrTarget)
        .addImm(0);
    }

    if (Br)
      Br->getOperand(0).setMBB(BrCond->getOperand(1).getMBB());

    MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
    MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
    MI.eraseFromParent();
    BrCond->eraseFromParent();
    return true;
  }

  return false;
}
case Intrinsic::amdgcn_loop: {
  MachineInstr *Br = nullptr;
  if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI, Br)) {
    const SIRegisterInfo *TRI
      = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());

    B.setInstr(*BrCond);

    MachineBasicBlock *BrTarget = BrCond->getOperand(1).getMBB();
    if (Br)
      BrTarget = Br->getOperand(0).getMBB();

    Register Reg = MI.getOperand(2).getReg();
    B.buildInstr(AMDGPU::SI_LOOP)
      .addUse(Reg)
      .addMBB(BrTarget);

    if (Br)
      Br->getOperand(0).setMBB(BrCond->getOperand(1).getMBB());

    MI.eraseFromParent();
    BrCond->eraseFromParent();
    MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
    return true;
  }

  return false;
}
case Intrinsic::amdgcn_kernarg_segment_ptr:
  return legalizePreloadedArgIntrin(
    MI, MRI, B, AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
case Intrinsic::amdgcn_implicitarg_ptr:
  return legalizeImplicitArgPtr(MI, MRI, B);
2
←
Calling 'AMDGPULegalizerInfo::legalizeImplicitArgPtr'→
case Intrinsic::amdgcn_workitem_id_x:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::WORKITEM_ID_X);
case Intrinsic::amdgcn_workitem_id_y:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
case Intrinsic::amdgcn_workitem_id_z:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
case Intrinsic::amdgcn_workgroup_id_x:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
case Intrinsic::amdgcn_workgroup_id_y:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
case Intrinsic::amdgcn_workgroup_id_z:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
case Intrinsic::amdgcn_dispatch_ptr:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::DISPATCH_PTR);
case Intrinsic::amdgcn_queue_ptr:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::QUEUE_PTR);
case Intrinsic::amdgcn_implicit_buffer_ptr:
  return legalizePreloadedArgIntrin(
    MI, MRI, B, AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
case Intrinsic::amdgcn_dispatch_id:
  return legalizePreloadedArgIntrin(MI, MRI, B,
                                    AMDGPUFunctionArgInfo::DISPATCH_ID);
case Intrinsic::amdgcn_fdiv_fast:
  return legalizeFDIVFastIntrin(MI, MRI, B);
case Intrinsic::amdgcn_is_shared:
  return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::LOCAL_ADDRESS);
case Intrinsic::amdgcn_is_private:
  return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::PRIVATE_ADDRESS);
case Intrinsic::amdgcn_wavefrontsize: {
  B.setInstr(MI);
  B.buildConstant(MI.getOperand(0), ST.getWavefrontSize());
  MI.eraseFromParent();
  return true;
}
case Intrinsic::amdgcn_s_buffer_load:
  return legalizeSBufferLoad(MI, B, Observer);
case Intrinsic::amdgcn_raw_buffer_store:
case Intrinsic::amdgcn_struct_buffer_store:
  return legalizeBufferStore(MI, MRI, B, false, false);
case Intrinsic::amdgcn_raw_buffer_store_format:
case Intrinsic::amdgcn_struct_buffer_store_format:
  return legalizeBufferStore(MI, MRI, B, false, true);
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_struct_tbuffer_store:
  return legalizeBufferStore(MI, MRI, B, true, true);
case Intrinsic::amdgcn_raw_buffer_load:
case Intrinsic::amdgcn_struct_buffer_load:
  return legalizeBufferLoad(MI, MRI, B, false, false);
case Intrinsic::amdgcn_raw_buffer_load_format:
case Intrinsic::amdgcn_struct_buffer_load_format:
  return legalizeBufferLoad(MI, MRI, B, true, false);
case Intrinsic::amdgcn_raw_tbuffer_load:
case Intrinsic::amdgcn_struct_tbuffer_load:
  return legalizeBufferLoad(MI, MRI, B, true, true);
case Intrinsic::amdgcn_raw_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_buffer_atomic_swap:
case Intrinsic::amdgcn_raw_buffer_atomic_add:
case Intrinsic::amdgcn_struct_buffer_atomic_add:
case Intrinsic::amdgcn_raw_buffer_atomic_sub:
case Intrinsic::amdgcn_struct_buffer_atomic_sub:
case Intrinsic::amdgcn_raw_buffer_atomic_smin:
case Intrinsic::amdgcn_struct_buffer_atomic_smin:
case Intrinsic::amdgcn_raw_buffer_atomic_umin:
case Intrinsic::amdgcn_struct_buffer_atomic_umin:
case Intrinsic::amdgcn_raw_buffer_atomic_smax:
case Intrinsic::amdgcn_struct_buffer_atomic_smax:
case Intrinsic::amdgcn_raw_buffer_atomic_umax:
case Intrinsic::amdgcn_struct_buffer_atomic_umax:
case Intrinsic::amdgcn_raw_buffer_atomic_and:
case Intrinsic::amdgcn_struct_buffer_atomic_and:
case Intrinsic::amdgcn_raw_buffer_atomic_or:
case Intrinsic::amdgcn_struct_buffer_atomic_or:
case Intrinsic::amdgcn_raw_buffer_atomic_xor:
case Intrinsic::amdgcn_struct_buffer_atomic_xor:
case Intrinsic::amdgcn_raw_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_buffer_atomic_inc:
case Intrinsic::amdgcn_raw_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_buffer_atomic_dec:
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
  return legalizeBufferAtomic(MI, B, IntrID);
case Intrinsic::amdgcn_atomic_inc:
  return legalizeAtomicIncDec(MI, B, true);
case Intrinsic::amdgcn_atomic_dec:
  return legalizeAtomicIncDec(MI, B, false);
default: {
  if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
          AMDGPU::getImageDimIntrinsicInfo(IntrID))
    return legalizeImageIntrinsic(MI, B, Observer, ImageDimIntr);
  return true;
}
}

return true;
3744}

←

/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h

→

1//==- AMDGPUArgumentrUsageInfo.h - Function Arg Usage Info -------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//

9#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUARGUMENTUSAGEINFO_H
10#define LLVM_LIB_TARGET_AMDGPU_AMDGPUARGUMENTUSAGEINFO_H

12#include "llvm/ADT/DenseMap.h"
13#include "llvm/CodeGen/Register.h"
14#include "llvm/IR/Function.h"
15#include "llvm/Pass.h"

17namespace llvm {

19class Function;
20class raw_ostream;
21class GCNSubtarget;
22class TargetMachine;
23class TargetRegisterClass;
24class TargetRegisterInfo;

26struct ArgDescriptor {
27private:
friend struct AMDGPUFunctionArgInfo;
friend class AMDGPUArgumentUsageInfo;

union {
  Register Reg;
  unsigned StackOffset;
};

// Bitmask to locate argument within the register.
unsigned Mask;

bool IsStack : 1;
bool IsSet : 1;

42public:
ArgDescriptor(unsigned Val = 0, unsigned Mask = ~0u,
              bool IsStack = false, bool IsSet = false)
  : Reg(Val), Mask(Mask), IsStack(IsStack), IsSet(IsSet) {}

static ArgDescriptor createRegister(Register Reg, unsigned Mask = ~0u) {
  return ArgDescriptor(Reg, Mask, false, true);
}

static ArgDescriptor createStack(unsigned Offset, unsigned Mask = ~0u) {
  return ArgDescriptor(Offset, Mask, true, true);
}

static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask) {
  return ArgDescriptor(Arg.Reg, Mask, Arg.IsStack, Arg.IsSet);
}

bool isSet() const {
  return IsSet;
}

explicit operator bool() const {
  return isSet();
}

bool isRegister() const {
  return !IsStack;
}

Register getRegister() const {
  assert(!IsStack)((!IsStack) ? static_cast<void> (0) : __assert_fail ("!IsStack"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h"
, 72, __PRETTY_FUNCTION__));
  return Reg;
}

unsigned getStackOffset() const {
  assert(IsStack)((IsStack) ? static_cast<void> (0) : __assert_fail ("IsStack"
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h"
, 77, __PRETTY_FUNCTION__));
  return StackOffset;
}

unsigned getMask() const {
  return Mask;
}

bool isMasked() const {
  return Mask != ~0u;
12
←
Assuming the condition is true→
13
←
Returning the value 1, which participates in a condition later→
}

void print(raw_ostream &OS, const TargetRegisterInfo *TRI = nullptr) const;
90};

92inline raw_ostream &operator<<(raw_ostream &OS, const ArgDescriptor &Arg) {
Arg.print(OS);
return OS;
95}

97struct AMDGPUFunctionArgInfo {
enum PreloadedValue {
  // SGPRS:
  PRIVATE_SEGMENT_BUFFER = 0,
  DISPATCH_PTR        =  1,
  QUEUE_PTR           =  2,
  KERNARG_SEGMENT_PTR =  3,
  DISPATCH_ID         =  4,
  FLAT_SCRATCH_INIT   =  5,
  WORKGROUP_ID_X      = 10,
  WORKGROUP_ID_Y      = 11,
  WORKGROUP_ID_Z      = 12,
  PRIVATE_SEGMENT_WAVE_BYTE_OFFSET = 14,
  IMPLICIT_BUFFER_PTR = 15,
  IMPLICIT_ARG_PTR = 16,

  // VGPRS:
  WORKITEM_ID_X       = 17,
  WORKITEM_ID_Y       = 18,
  WORKITEM_ID_Z       = 19,
  FIRST_VGPR_VALUE    = WORKITEM_ID_X
};

// Kernel input registers setup for the HSA ABI in allocation order.

// User SGPRs in kernels
// XXX - Can these require argument spills?
ArgDescriptor PrivateSegmentBuffer;
ArgDescriptor DispatchPtr;
ArgDescriptor QueuePtr;
ArgDescriptor KernargSegmentPtr;
ArgDescriptor DispatchID;
ArgDescriptor FlatScratchInit;
ArgDescriptor PrivateSegmentSize;

// System SGPRs in kernels.
ArgDescriptor WorkGroupIDX;
ArgDescriptor WorkGroupIDY;
ArgDescriptor WorkGroupIDZ;
ArgDescriptor WorkGroupInfo;
ArgDescriptor PrivateSegmentWaveByteOffset;

// Pointer with offset from kernargsegmentptr to where special ABI arguments
// are passed to callable functions.
ArgDescriptor ImplicitArgPtr;

// Input registers for non-HSA ABI
ArgDescriptor ImplicitBufferPtr = 0;

// VGPRs inputs. These are always v0, v1 and v2 for entry functions.
ArgDescriptor WorkItemIDX;
ArgDescriptor WorkItemIDY;
ArgDescriptor WorkItemIDZ;

std::pair<const ArgDescriptor *, const TargetRegisterClass *>
getPreloadedValue(PreloadedValue Value) const;
153};

155class AMDGPUArgumentUsageInfo : public ImmutablePass {
156private:
static const AMDGPUFunctionArgInfo ExternFunctionInfo;
DenseMap<const Function *, AMDGPUFunctionArgInfo> ArgInfoMap;

160public:
static char ID;

AMDGPUArgumentUsageInfo() : ImmutablePass(ID) { }

void getAnalysisUsage(AnalysisUsage &AU) const override {
  AU.setPreservesAll();
}

bool doInitialization(Module &M) override;
bool doFinalization(Module &M) override;

void print(raw_ostream &OS, const Module *M = nullptr) const override;

void setFuncArgInfo(const Function &F, const AMDGPUFunctionArgInfo &ArgInfo) {
  ArgInfoMap[&F] = ArgInfo;
}

const AMDGPUFunctionArgInfo &lookupFuncArgInfo(const Function &F) const {
  auto I = ArgInfoMap.find(&F);
  if (I == ArgInfoMap.end()) {
    assert(F.isDeclaration())((F.isDeclaration()) ? static_cast<void> (0) : __assert_fail
 ("F.isDeclaration()", "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h"
, 181, __PRETTY_FUNCTION__));
    return ExternFunctionInfo;
  }

  return I->second;
}
187};

189} // end namespace llvm

191#endif

←

/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains some functions that are useful for math stuff.
10//
11//===----------------------------------------------------------------------===//
12 
13#ifndef LLVM_SUPPORT_MATHEXTRAS_H
14#define LLVM_SUPPORT_MATHEXTRAS_H
15 
16#include "llvm/Support/Compiler.h"
17#include <algorithm>
18#include <cassert>
19#include <climits>
20#include <cmath>
21#include <cstdint>
22#include <cstring>
23#include <limits>
24#include <type_traits>
25 
26#ifdef __ANDROID_NDK__
27#include <android/api-level.h>
28#endif
29 
30#ifdef _MSC_VER
31// Declare these intrinsics manually rather including intrin.h. It's very
32// expensive, and MathExtras.h is popular.
33// #include <intrin.h>
34extern "C" {
35unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
36unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
37unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
38unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
39}
40#endif
41 
42namespace llvm {
43 
44/// The behavior an operation has on an input of 0.
45enum ZeroBehavior {
46  /// The returned value is undefined.
47  ZB_Undefined,
48  /// The returned value is numeric_limits<T>::max()
49  ZB_Max,
50  /// The returned value is numeric_limits<T>::digits
51  ZB_Width
52};
53 
54/// Mathematical constants.
55namespace numbers {
56// TODO: Track C++20 std::numbers.
57// TODO: Favor using the hexadecimal FP constants (requires C++17).
58constexpr double e          = 2.7182818284590452354, // (0x1.5bf0a8b145749P+1) https://oeis.org/A001113
59                 egamma     = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620
60                 ln2        = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162
61                 ln10       = 2.3025850929940456840, // (0x1.24bb1bbb55516P+1) https://oeis.org/A002392
62                 log2e      = 1.4426950408889634074, // (0x1.71547652b82feP+0)
63                 log10e     = .43429448190325182765, // (0x1.bcb7b1526e50eP-2)
64                 pi         = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796
65                 inv_pi     = .31830988618379067154, // (0x1.45f306bc9c883P-2) https://oeis.org/A049541
66                 sqrtpi     = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161
67                 inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197
68                 sqrt2      = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219
69                 inv_sqrt2  = .70710678118654752440, // (0x1.6a09e667f3bcdP-1)
70                 sqrt3      = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194
71                 inv_sqrt3  = .57735026918962576451, // (0x1.279a74590331cP-1)
72                 phi        = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622
73constexpr float ef          = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113
74                egammaf     = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620
75                ln2f        = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162
76                ln10f       = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392
77                log2ef      = 1.44269504F, // (0x1.715476P+0)
78                log10ef     = .434294482F, // (0x1.bcb7b2P-2)
79                pif         = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796
80                inv_pif     = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541
81                sqrtpif     = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161
82                inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197
83                sqrt2f      = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193
84                inv_sqrt2f  = .707106781F, // (0x1.6a09e6P-1)
85                sqrt3f      = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194
86                inv_sqrt3f  = .577350269F, // (0x1.279a74P-1)
87                phif        = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
88} // namespace numbers
89 
90namespace detail {
91template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
92  static unsigned count(T Val, ZeroBehavior) {
93    if (!Val)
94      return std::numeric_limits<T>::digits;
95    if (Val & 0x1)
96      return 0;
97 
98    // Bisection method.
99    unsigned ZeroBits = 0;
100    T Shift = std::numeric_limits<T>::digits >> 1;
101    T Mask = std::numeric_limits<T>::max() >> Shift;
102    while (Shift) {
103      if ((Val & Mask) == 0) {
104        Val >>= Shift;
105        ZeroBits |= Shift;
106      }
107      Shift >>= 1;
108      Mask >>= Shift;
109    }
110    return ZeroBits;
111  }
112};
113 
114#if defined(__GNUC__4) || defined(_MSC_VER)
115template <typename T> struct TrailingZerosCounter<T, 4> {
116  static unsigned count(T Val, ZeroBehavior ZB) {
117    if (ZB17.1
'ZB' is not equal to ZB_Undefined
17.1
'ZB' is not equal to ZB_Undefined
17.1
'ZB' is not equal to ZB_Undefined
 != ZB_Undefined && Val == 0)
18
←
Assuming 'Val' is equal to 0→
19
←
Taking true branch→
118      return 32;
20
←
Returning the value 32→
119 
120#if __has_builtin(__builtin_ctz)1 || defined(__GNUC__4)
121    return __builtin_ctz(Val);
122#elif defined(_MSC_VER)
123    unsigned long Index;
124    _BitScanForward(&Index, Val);
125    return Index;
126#endif
127  }
128};
129 
130#if !defined(_MSC_VER) || defined(_M_X64)
131template <typename T> struct TrailingZerosCounter<T, 8> {
132  static unsigned count(T Val, ZeroBehavior ZB) {
133    if (ZB != ZB_Undefined && Val == 0)
134      return 64;
135 
136#if __has_builtin(__builtin_ctzll)1 || defined(__GNUC__4)
137    return __builtin_ctzll(Val);
138#elif defined(_MSC_VER)
139    unsigned long Index;
140    _BitScanForward64(&Index, Val);
141    return Index;
142#endif
143  }
144};
145#endif
146#endif
147} // namespace detail
148 
149/// Count number of 0's from the least significant bit to the most
150///   stopping at the first 1.
151///
152/// Only unsigned integral types are allowed.
153///
154/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
155///   valid arguments.
156template <typename T>
157unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
158  static_assert(std::numeric_limits<T>::is_integer &&
159                    !std::numeric_limits<T>::is_signed,
160                "Only unsigned integral types are allowed.");
161  return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
17
←
Calling 'TrailingZerosCounter::count'→
21
←
Returning from 'TrailingZerosCounter::count'→
22
←
Returning the value 32→
162}
163 
164namespace detail {
165template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
166  static unsigned count(T Val, ZeroBehavior) {
167    if (!Val)
168      return std::numeric_limits<T>::digits;
169 
170    // Bisection method.
171    unsigned ZeroBits = 0;
172    for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
173      T Tmp = Val >> Shift;
174      if (Tmp)
175        Val = Tmp;
176      else
177        ZeroBits |= Shift;
178    }
179    return ZeroBits;
180  }
181};
182 
183#if defined(__GNUC__4) || defined(_MSC_VER)
184template <typename T> struct LeadingZerosCounter<T, 4> {
185  static unsigned count(T Val, ZeroBehavior ZB) {
186    if (ZB != ZB_Undefined && Val == 0)
187      return 32;
188 
189#if __has_builtin(__builtin_clz)1 || defined(__GNUC__4)
190    return __builtin_clz(Val);
191#elif defined(_MSC_VER)
192    unsigned long Index;
193    _BitScanReverse(&Index, Val);
194    return Index ^ 31;
195#endif
196  }
197};
198 
199#if !defined(_MSC_VER) || defined(_M_X64)
200template <typename T> struct LeadingZerosCounter<T, 8> {
201  static unsigned count(T Val, ZeroBehavior ZB) {
202    if (ZB != ZB_Undefined && Val == 0)
203      return 64;
204 
205#if __has_builtin(__builtin_clzll)1 || defined(__GNUC__4)
206    return __builtin_clzll(Val);
207#elif defined(_MSC_VER)
208    unsigned long Index;
209    _BitScanReverse64(&Index, Val);
210    return Index ^ 63;
211#endif
212  }
213};
214#endif
215#endif
216} // namespace detail
217 
218/// Count number of 0's from the most significant bit to the least
219///   stopping at the first 1.
220///
221/// Only unsigned integral types are allowed.
222///
223/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
224///   valid arguments.
225template <typename T>
226unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
227  static_assert(std::numeric_limits<T>::is_integer &&
228                    !std::numeric_limits<T>::is_signed,
229                "Only unsigned integral types are allowed.");
230  return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
231}
232 
233/// Get the index of the first set bit starting from the least
234///   significant bit.
235///
236/// Only unsigned integral types are allowed.
237///
238/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
239///   valid arguments.
240template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
241  if (ZB == ZB_Max && Val == 0)
242    return std::numeric_limits<T>::max();
243 
244  return countTrailingZeros(Val, ZB_Undefined);
245}
246 
247/// Create a bitmask with the N right-most bits set to 1, and all other
248/// bits set to 0.  Only unsigned types are allowed.
249template <typename T> T maskTrailingOnes(unsigned N) {
250  static_assert(std::is_unsigned<T>::value, "Invalid type!");
251  const unsigned Bits = CHAR_BIT8 * sizeof(T);
252  assert(N <= Bits && "Invalid bit index")((N <= Bits && "Invalid bit index") ? static_cast<
void> (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 252, __PRETTY_FUNCTION__));
253  return N == 0 ? 0 : (T(-1) >> (Bits - N));
254}
255 
256/// Create a bitmask with the N left-most bits set to 1, and all other
257/// bits set to 0.  Only unsigned types are allowed.
258template <typename T> T maskLeadingOnes(unsigned N) {
259  return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
260}
261 
262/// Create a bitmask with the N right-most bits set to 0, and all other
263/// bits set to 1.  Only unsigned types are allowed.
264template <typename T> T maskTrailingZeros(unsigned N) {
265  return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
266}
267 
268/// Create a bitmask with the N left-most bits set to 0, and all other
269/// bits set to 1.  Only unsigned types are allowed.
270template <typename T> T maskLeadingZeros(unsigned N) {
271  return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
272}
273 
274/// Get the index of the last set bit starting from the least
275///   significant bit.
276///
277/// Only unsigned integral types are allowed.
278///
279/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
280///   valid arguments.
281template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
282  if (ZB == ZB_Max && Val == 0)
283    return std::numeric_limits<T>::max();
284 
285  // Use ^ instead of - because both gcc and llvm can remove the associated ^
286  // in the __builtin_clz intrinsic on x86.
287  return countLeadingZeros(Val, ZB_Undefined) ^
288         (std::numeric_limits<T>::digits - 1);
289}
290 
291/// Macro compressed bit reversal table for 256 bits.
292///
293/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
294static const unsigned char BitReverseTable256[256] = {
295#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
296#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
297#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
298  R6(0), R6(2), R6(1), R6(3)
299#undef R2
300#undef R4
301#undef R6
302};
303 
304/// Reverse the bits in \p Val.
305template <typename T>
306T reverseBits(T Val) {
307  unsigned char in[sizeof(Val)];
308  unsigned char out[sizeof(Val)];
309  std::memcpy(in, &Val, sizeof(Val));
310  for (unsigned i = 0; i < sizeof(Val); ++i)
311    out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
312  std::memcpy(&Val, out, sizeof(Val));
313  return Val;
314}
315 
316// NOTE: The following support functions use the _32/_64 extensions instead of
317// type overloading so that signed and unsigned integers can be used without
318// ambiguity.
319 
320/// Return the high 32 bits of a 64 bit value.
321constexpr inline uint32_t Hi_32(uint64_t Value) {
322  return static_cast<uint32_t>(Value >> 32);
323}
324 
325/// Return the low 32 bits of a 64 bit value.
326constexpr inline uint32_t Lo_32(uint64_t Value) {
327  return static_cast<uint32_t>(Value);
328}
329 
330/// Make a 64-bit integer from a high / low pair of 32-bit integers.
331constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
332  return ((uint64_t)High << 32) | (uint64_t)Low;
333}
334 
335/// Checks if an integer fits into the given bit width.
336template <unsigned N> constexpr inline bool isInt(int64_t x) {
337  return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
338}
339// Template specializations to get better code for common cases.
340template <> constexpr inline bool isInt<8>(int64_t x) {
341  return static_cast<int8_t>(x) == x;
342}
343template <> constexpr inline bool isInt<16>(int64_t x) {
344  return static_cast<int16_t>(x) == x;
345}
346template <> constexpr inline bool isInt<32>(int64_t x) {
347  return static_cast<int32_t>(x) == x;
348}
349 
350/// Checks if a signed integer is an N bit number shifted left by S.
351template <unsigned N, unsigned S>
352constexpr inline bool isShiftedInt(int64_t x) {
353  static_assert(
354      N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
355  static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
356  return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
357}
358 
359/// Checks if an unsigned integer fits into the given bit width.
360///
361/// This is written as two functions rather than as simply
362///
363///   return N >= 64 || X < (UINT64_C(1) << N);
364///
365/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
366/// left too many places.
367template <unsigned N>
368constexpr inline std::enable_if_t<(N < 64), bool> isUInt(uint64_t X) {
369  static_assert(N > 0, "isUInt<0> doesn't make sense");
370  return X < (UINT64_C(1)1UL << (N));
371}
372template <unsigned N>
373constexpr inline std::enable_if_t<N >= 64, bool> isUInt(uint64_t X) {
374  return true;
375}
376 
377// Template specializations to get better code for common cases.
378template <> constexpr inline bool isUInt<8>(uint64_t x) {
379  return static_cast<uint8_t>(x) == x;
380}
381template <> constexpr inline bool isUInt<16>(uint64_t x) {
382  return static_cast<uint16_t>(x) == x;
383}
384template <> constexpr inline bool isUInt<32>(uint64_t x) {
385  return static_cast<uint32_t>(x) == x;
386}
387 
388/// Checks if a unsigned integer is an N bit number shifted left by S.
389template <unsigned N, unsigned S>
390constexpr inline bool isShiftedUInt(uint64_t x) {
391  static_assert(
392      N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
393  static_assert(N + S <= 64,
394                "isShiftedUInt<N, S> with N + S > 64 is too wide.");
395  // Per the two static_asserts above, S must be strictly less than 64.  So
396  // 1 << S is not undefined behavior.
397  return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
398}
399 
400/// Gets the maximum value for a N-bit unsigned integer.
401inline uint64_t maxUIntN(uint64_t N) {
402  assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 402, __PRETTY_FUNCTION__));
403 
404  // uint64_t(1) << 64 is undefined behavior, so we can't do
405  //   (uint64_t(1) << N) - 1
406  // without checking first that N != 64.  But this works and doesn't have a
407  // branch.
408  return UINT64_MAX(18446744073709551615UL) >> (64 - N);
409}
410 
411/// Gets the minimum value for a N-bit signed integer.
412inline int64_t minIntN(int64_t N) {
413  assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 413, __PRETTY_FUNCTION__));
414 
415  return -(UINT64_C(1)1UL<<(N-1));
416}
417 
418/// Gets the maximum value for a N-bit signed integer.
419inline int64_t maxIntN(int64_t N) {
420  assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 420, __PRETTY_FUNCTION__));
421 
422  // This relies on two's complement wraparound when N == 64, so we convert to
423  // int64_t only at the very end to avoid UB.
424  return (UINT64_C(1)1UL << (N - 1)) - 1;
425}
426 
427/// Checks if an unsigned integer fits into the given (dynamic) bit width.
428inline bool isUIntN(unsigned N, uint64_t x) {
429  return N >= 64 || x <= maxUIntN(N);
430}
431 
432/// Checks if an signed integer fits into the given (dynamic) bit width.
433inline bool isIntN(unsigned N, int64_t x) {
434  return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
435}
436 
437/// Return true if the argument is a non-empty sequence of ones starting at the
438/// least significant bit with the remainder zero (32 bit version).
439/// Ex. isMask_32(0x0000FFFFU) == true.
440constexpr inline bool isMask_32(uint32_t Value) {
441  return Value && ((Value + 1) & Value) == 0;
442}
443 
444/// Return true if the argument is a non-empty sequence of ones starting at the
445/// least significant bit with the remainder zero (64 bit version).
446constexpr inline bool isMask_64(uint64_t Value) {
447  return Value && ((Value + 1) & Value) == 0;
448}
449 
450/// Return true if the argument contains a non-empty sequence of ones with the
451/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
452constexpr inline bool isShiftedMask_32(uint32_t Value) {
453  return Value && isMask_32((Value - 1) | Value);
454}
455 
456/// Return true if the argument contains a non-empty sequence of ones with the
457/// remainder zero (64 bit version.)
458constexpr inline bool isShiftedMask_64(uint64_t Value) {
459  return Value && isMask_64((Value - 1) | Value);
460}
461 
462/// Return true if the argument is a power of two > 0.
463/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
464constexpr inline bool isPowerOf2_32(uint32_t Value) {
465  return Value && !(Value & (Value - 1));
466}
467 
468/// Return true if the argument is a power of two > 0 (64 bit edition.)
469constexpr inline bool isPowerOf2_64(uint64_t Value) {
470  return Value && !(Value & (Value - 1));
471}
472 
473/// Count the number of ones from the most significant bit to the first
474/// zero bit.
475///
476/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
477/// Only unsigned integral types are allowed.
478///
479/// \param ZB the behavior on an input of all ones. Only ZB_Width and
480/// ZB_Undefined are valid arguments.
481template <typename T>
482unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
483  static_assert(std::numeric_limits<T>::is_integer &&
484                    !std::numeric_limits<T>::is_signed,
485                "Only unsigned integral types are allowed.");
486  return countLeadingZeros<T>(~Value, ZB);
487}
488 
489/// Count the number of ones from the least significant bit to the first
490/// zero bit.
491///
492/// Ex. countTrailingOnes(0x00FF00FF) == 8.
493/// Only unsigned integral types are allowed.
494///
495/// \param ZB the behavior on an input of all ones. Only ZB_Width and
496/// ZB_Undefined are valid arguments.
497template <typename T>
498unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
499  static_assert(std::numeric_limits<T>::is_integer &&
500                    !std::numeric_limits<T>::is_signed,
501                "Only unsigned integral types are allowed.");
502  return countTrailingZeros<T>(~Value, ZB);
503}
504 
505namespace detail {
506template <typename T, std::size_t SizeOfT> struct PopulationCounter {
507  static unsigned count(T Value) {
508    // Generic version, forward to 32 bits.
509    static_assert(SizeOfT <= 4, "Not implemented!");
510#if defined(__GNUC__4)
511    return __builtin_popcount(Value);
512#else
513    uint32_t v = Value;
514    v = v - ((v >> 1) & 0x55555555);
515    v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
516    return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
517#endif
518  }
519};
520 
521template <typename T> struct PopulationCounter<T, 8> {
522  static unsigned count(T Value) {
523#if defined(__GNUC__4)
524    return __builtin_popcountll(Value);
525#else
526    uint64_t v = Value;
527    v = v - ((v >> 1) & 0x5555555555555555ULL);
528    v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
529    v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
530    return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
531#endif
532  }
533};
534} // namespace detail
535 
536/// Count the number of set bits in a value.
537/// Ex. countPopulation(0xF000F000) = 8
538/// Returns 0 if the word is zero.
539template <typename T>
540inline unsigned countPopulation(T Value) {
541  static_assert(std::numeric_limits<T>::is_integer &&
542                    !std::numeric_limits<T>::is_signed,
543                "Only unsigned integral types are allowed.");
544  return detail::PopulationCounter<T, sizeof(T)>::count(Value);
545}
546 
547/// Compile time Log2.
548/// Valid only for positive powers of two.
549template <size_t kValue> constexpr inline size_t CTLog2() {
550  static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue),
551                "Value is not a valid power of 2");
552  return 1 + CTLog2<kValue / 2>();
553}
554 
555template <> constexpr inline size_t CTLog2<1>() { return 0; }
556 
557/// Return the log base 2 of the specified value.
558inline double Log2(double Value) {
559#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
560  return __builtin_log(Value) / __builtin_log(2.0);
561#else
562  return log2(Value);
563#endif
564}
565 
566/// Return the floor log base 2 of the specified value, -1 if the value is zero.
567/// (32 bit edition.)
568/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
569inline unsigned Log2_32(uint32_t Value) {
570  return 31 - countLeadingZeros(Value);
571}
572 
573/// Return the floor log base 2 of the specified value, -1 if the value is zero.
574/// (64 bit edition.)
575inline unsigned Log2_64(uint64_t Value) {
576  return 63 - countLeadingZeros(Value);
577}
578 
579/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
580/// (32 bit edition).
581/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
582inline unsigned Log2_32_Ceil(uint32_t Value) {
583  return 32 - countLeadingZeros(Value - 1);
584}
585 
586/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
587/// (64 bit edition.)
588inline unsigned Log2_64_Ceil(uint64_t Value) {
589  return 64 - countLeadingZeros(Value - 1);
590}
591 
592/// Return the greatest common divisor of the values using Euclid's algorithm.
593template <typename T>
594inline T greatestCommonDivisor(T A, T B) {
595  while (B) {
596    T Tmp = B;
597    B = A % B;
598    A = Tmp;
599  }
600  return A;
601}
602 
603inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
604  return greatestCommonDivisor<uint64_t>(A, B);
605}
606 
607/// This function takes a 64-bit integer and returns the bit equivalent double.
608inline double BitsToDouble(uint64_t Bits) {
609  double D;
610  static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
611  memcpy(&D, &Bits, sizeof(Bits));
612  return D;
613}
614 
615/// This function takes a 32-bit integer and returns the bit equivalent float.
616inline float BitsToFloat(uint32_t Bits) {
617  float F;
618  static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
619  memcpy(&F, &Bits, sizeof(Bits));
620  return F;
621}
622 
623/// This function takes a double and returns the bit equivalent 64-bit integer.
624/// Note that copying doubles around changes the bits of NaNs on some hosts,
625/// notably x86, so this routine cannot be used if these bits are needed.
626inline uint64_t DoubleToBits(double Double) {
627  uint64_t Bits;
628  static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
629  memcpy(&Bits, &Double, sizeof(Double));
630  return Bits;
631}
632 
633/// This function takes a float and returns the bit equivalent 32-bit integer.
634/// Note that copying floats around changes the bits of NaNs on some hosts,
635/// notably x86, so this routine cannot be used if these bits are needed.
636inline uint32_t FloatToBits(float Float) {
637  uint32_t Bits;
638  static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
639  memcpy(&Bits, &Float, sizeof(Float));
640  return Bits;
641}
642 
643/// A and B are either alignments or offsets. Return the minimum alignment that
644/// may be assumed after adding the two together.
645constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
646  // The largest power of 2 that divides both A and B.
647  //
648  // Replace "-Value" by "1+~Value" in the following commented code to avoid
649  // MSVC warning C4146
650  //    return (A | B) & -(A | B);
651  return (A | B) & (1 + ~(A | B));
652}
653 
654/// Returns the next power of two (in 64-bits) that is strictly greater than A.
655/// Returns zero on overflow.
656inline uint64_t NextPowerOf2(uint64_t A) {
657  A |= (A >> 1);
658  A |= (A >> 2);
659  A |= (A >> 4);
660  A |= (A >> 8);
661  A |= (A >> 16);
662  A |= (A >> 32);
663  return A + 1;
664}
665 
666/// Returns the power of two which is less than or equal to the given value.
667/// Essentially, it is a floor operation across the domain of powers of two.
668inline uint64_t PowerOf2Floor(uint64_t A) {
669  if (!A) return 0;
670  return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
671}
672 
673/// Returns the power of two which is greater than or equal to the given value.
674/// Essentially, it is a ceil operation across the domain of powers of two.
675inline uint64_t PowerOf2Ceil(uint64_t A) {
676  if (!A)
677    return 0;
678  return NextPowerOf2(A - 1);
679}
680 
681/// Returns the next integer (mod 2**64) that is greater than or equal to
682/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
683///
684/// If non-zero \p Skew is specified, the return value will be a minimal
685/// integer that is greater than or equal to \p Value and equal to
686/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
687/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
688///
689/// Examples:
690/// \code
691///   alignTo(5, 8) = 8
692///   alignTo(17, 8) = 24
693///   alignTo(~0LL, 8) = 0
694///   alignTo(321, 255) = 510
695///
696///   alignTo(5, 8, 7) = 7
697///   alignTo(17, 8, 1) = 17
698///   alignTo(~0LL, 8, 3) = 3
699///   alignTo(321, 255, 42) = 552
700/// \endcode
701inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
702  assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 702, __PRETTY_FUNCTION__));
703  Skew %= Align;
704  return (Value + Align - 1 - Skew) / Align * Align + Skew;
705}
706 
707/// Returns the next integer (mod 2**64) that is greater than or equal to
708/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
709template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
710  static_assert(Align != 0u, "Align must be non-zero");
711  return (Value + Align - 1) / Align * Align;
712}
713 
714/// Returns the integer ceil(Numerator / Denominator).
715inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
716  return alignTo(Numerator, Denominator) / Denominator;
717}
718 
719/// Returns the integer nearest(Numerator / Denominator).
720inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) {
721  return (Numerator + (Denominator / 2)) / Denominator;
722}
723 
724/// Returns the largest uint64_t less than or equal to \p Value and is
725/// \p Skew mod \p Align. \p Align must be non-zero
726inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
727  assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 727, __PRETTY_FUNCTION__));
728  Skew %= Align;
729  return (Value - Skew) / Align * Align + Skew;
730}
731 
732/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
733/// Requires 0 < B <= 32.
734template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
735  static_assert(B > 0, "Bit width can't be 0.");
736  static_assert(B <= 32, "Bit width out of range.");
737  return int32_t(X << (32 - B)) >> (32 - B);
738}
739 
740/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
741/// Requires 0 < B < 32.
742inline int32_t SignExtend32(uint32_t X, unsigned B) {
743  assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 743, __PRETTY_FUNCTION__));
744  assert(B <= 32 && "Bit width out of range.")((B <= 32 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 744, __PRETTY_FUNCTION__));
745  return int32_t(X << (32 - B)) >> (32 - B);
746}
747 
748/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
749/// Requires 0 < B < 64.
750template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
751  static_assert(B > 0, "Bit width can't be 0.");
752  static_assert(B <= 64, "Bit width out of range.");
753  return int64_t(x << (64 - B)) >> (64 - B);
754}
755 
756/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
757/// Requires 0 < B < 64.
758inline int64_t SignExtend64(uint64_t X, unsigned B) {
759  assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 759, __PRETTY_FUNCTION__));
760  assert(B <= 64 && "Bit width out of range.")((B <= 64 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-11~++20200304121622+a8706b22a62/llvm/include/llvm/Support/MathExtras.h"
, 760, __PRETTY_FUNCTION__));
761  return int64_t(X << (64 - B)) >> (64 - B);
762}
763 
764/// Subtract two unsigned integers, X and Y, of type T and return the absolute
765/// value of the result.
766template <typename T>
767std::enable_if_t<std::is_unsigned<T>::value, T> AbsoluteDifference(T X, T Y) {
768  return std::max(X, Y) - std::min(X, Y);
769}
770 
771/// Add two unsigned integers, X and Y, of type T.  Clamp the result to the
772/// maximum representable value of T on overflow.  ResultOverflowed indicates if
773/// the result is larger than the maximum representable value of type T.
774template <typename T>
775std::enable_if_t<std::is_unsigned<T>::value, T>
776SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
777  bool Dummy;
778  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
779  // Hacker's Delight, p. 29
780  T Z = X + Y;
781  Overflowed = (Z < X || Z < Y);
782  if (Overflowed)
783    return std::numeric_limits<T>::max();
784  else
785    return Z;
786}
787 
788/// Multiply two unsigned integers, X and Y, of type T.  Clamp the result to the
789/// maximum representable value of T on overflow.  ResultOverflowed indicates if
790/// the result is larger than the maximum representable value of type T.
791template <typename T>
792std::enable_if_t<std::is_unsigned<T>::value, T>
793SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
794  bool Dummy;
795  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
796 
797  // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
798  // because it fails for uint16_t (where multiplication can have undefined
799  // behavior due to promotion to int), and requires a division in addition
800  // to the multiplication.
801 
802  Overflowed = false;
803 
804  // Log2(Z) would be either Log2Z or Log2Z + 1.
805  // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
806  // will necessarily be less than Log2Max as desired.
807  int Log2Z = Log2_64(X) + Log2_64(Y);
808  const T Max = std::numeric_limits<T>::max();
809  int Log2Max = Log2_64(Max);
810  if (Log2Z < Log2Max) {
811    return X * Y;
812  }
813  if (Log2Z > Log2Max) {
814    Overflowed = true;
815    return Max;
816  }
817 
818  // We're going to use the top bit, and maybe overflow one
819  // bit past it. Multiply all but the bottom bit then add
820  // that on at the end.
821  T Z = (X >> 1) * Y;
822  if (Z & ~(Max >> 1)) {
823    Overflowed = true;
824    return Max;
825  }
826  Z <<= 1;
827  if (X & 1)
828    return SaturatingAdd(Z, Y, ResultOverflowed);
829 
830  return Z;
831}
832 
833/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
834/// the product. Clamp the result to the maximum representable value of T on
835/// overflow. ResultOverflowed indicates if the result is larger than the
836/// maximum representable value of type T.
837template <typename T>
838std::enable_if_t<std::is_unsigned<T>::value, T>
839SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
840  bool Dummy;
841  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
842 
843  T Product = SaturatingMultiply(X, Y, &Overflowed);
844  if (Overflowed)
845    return Product;
846 
847  return SaturatingAdd(A, Product, &Overflowed);
848}
849 
850/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
851extern const float huge_valf;
852 
853 
854/// Add two signed integers, computing the two's complement truncated result,
855/// returning true if overflow occured.
856template <typename T>
857std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) {
858#if __has_builtin(__builtin_add_overflow)1
859  return __builtin_add_overflow(X, Y, &Result);
860#else
861  // Perform the unsigned addition.
862  using U = std::make_unsigned_t<T>;
863  const U UX = static_cast<U>(X);
864  const U UY = static_cast<U>(Y);
865  const U UResult = UX + UY;
866 
867  // Convert to signed.
868  Result = static_cast<T>(UResult);
869 
870  // Adding two positive numbers should result in a positive number.
871  if (X > 0 && Y > 0)
872    return Result <= 0;
873  // Adding two negatives should result in a negative number.
874  if (X < 0 && Y < 0)
875    return Result >= 0;
876  return false;
877#endif
878}
879 
880/// Subtract two signed integers, computing the two's complement truncated
881/// result, returning true if an overflow ocurred.
882template <typename T>
883std::enable_if_t<std::is_signed<T>::value, T> SubOverflow(T X, T Y, T &Result) {
884#if __has_builtin(__builtin_sub_overflow)1
885  return __builtin_sub_overflow(X, Y, &Result);
886#else
887  // Perform the unsigned addition.
888  using U = std::make_unsigned_t<T>;
889  const U UX = static_cast<U>(X);
890  const U UY = static_cast<U>(Y);
891  const U UResult = UX - UY;
892 
893  // Convert to signed.
894  Result = static_cast<T>(UResult);
895 
896  // Subtracting a positive number from a negative results in a negative number.
897  if (X <= 0 && Y > 0)
898    return Result >= 0;
899  // Subtracting a negative number from a positive results in a positive number.
900  if (X >= 0 && Y < 0)
901    return Result <= 0;
902  return false;
903#endif
904}
905 
906/// Multiply two signed integers, computing the two's complement truncated
907/// result, returning true if an overflow ocurred.
908template <typename T>
909std::enable_if_t<std::is_signed<T>::value, T> MulOverflow(T X, T Y, T &Result) {
910  // Perform the unsigned multiplication on absolute values.
911  using U = std::make_unsigned_t<T>;
912  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
913  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
914  const U UResult = UX * UY;
915 
916  // Convert to signed.
917  const bool IsNegative = (X < 0) ^ (Y < 0);
918  Result = IsNegative ? (0 - UResult) : UResult;
919 
920  // If any of the args was 0, result is 0 and no overflow occurs.
921  if (UX == 0 || UY == 0)
922    return false;
923 
924  // UX and UY are in [1, 2^n], where n is the number of digits.
925  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
926  // positive) divided by an argument compares to the other.
927  if (IsNegative)
928    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
929  else
930    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
931}
932 
933} // End llvm namespace
934 
935#endif