docs/doxygen/X86LegalizerInfo_8cpp_source.html

//===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

/// \file

/// This file implements the targeting of the Machinelegalizer class for X86.

/// \todo This should be generated by TableGen.

//===----------------------------------------------------------------------===//


#include "X86LegalizerInfo.h"

#include "X86Subtarget.h"

#include "X86TargetMachine.h"

#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"

#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"

#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"

#include "llvm/CodeGen/MachineConstantPool.h"

#include "llvm/CodeGen/MachineFrameInfo.h"

#include "llvm/CodeGen/TargetOpcodes.h"

#include "llvm/CodeGen/ValueTypes.h"

#include "llvm/IR/DerivedTypes.h"

#include "llvm/IR/IntrinsicsX86.h"

#include "llvm/IR/Type.h"


using namespace llvm;

using namespace TargetOpcode;

using namespace LegalizeActions;

using namespace LegalityPredicates;


X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,

                                   const X86TargetMachine &TM)

    : Subtarget(STI) {


  bool Is64Bit = Subtarget.is64Bit();

  bool HasCMOV = Subtarget.canUseCMOV();

  bool HasSSE1 = Subtarget.hasSSE1();

  bool HasSSE2 = Subtarget.hasSSE2();

  bool HasSSE41 = Subtarget.hasSSE41();

  bool HasAVX = Subtarget.hasAVX();

  bool HasAVX2 = Subtarget.hasAVX2();

  bool HasAVX512 = Subtarget.hasAVX512();

  bool HasVLX = Subtarget.hasVLX();

  bool HasDQI = Subtarget.hasAVX512() && Subtarget.hasDQI();

  bool HasBWI = Subtarget.hasAVX512() && Subtarget.hasBWI();

  bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();

  bool HasPOPCNT = Subtarget.hasPOPCNT();

  bool HasLZCNT = Subtarget.hasLZCNT();

  bool HasBMI = Subtarget.hasBMI();


  const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0));

  const LLT s1 = LLT::scalar(1);

  const LLT s8 = LLT::scalar(8);

  const LLT s16 = LLT::scalar(16);

  const LLT s32 = LLT::scalar(32);

  const LLT s64 = LLT::scalar(64);

  const LLT s80 = LLT::scalar(80);

  const LLT s128 = LLT::scalar(128);

  const LLT sMaxScalar = Subtarget.is64Bit() ? s64 : s32;

  const LLT v2s32 = LLT::fixed_vector(2, 32);

  const LLT v4s8 = LLT::fixed_vector(4, 8);


  const LLT v16s8 = LLT::fixed_vector(16, 8);

  const LLT v8s16 = LLT::fixed_vector(8, 16);

  const LLT v4s32 = LLT::fixed_vector(4, 32);

  const LLT v2s64 = LLT::fixed_vector(2, 64);

  const LLT v2p0 = LLT::fixed_vector(2, p0);


  const LLT v32s8 = LLT::fixed_vector(32, 8);

  const LLT v16s16 = LLT::fixed_vector(16, 16);

  const LLT v8s32 = LLT::fixed_vector(8, 32);

  const LLT v4s64 = LLT::fixed_vector(4, 64);

  const LLT v4p0 = LLT::fixed_vector(4, p0);


  const LLT v64s8 = LLT::fixed_vector(64, 8);

  const LLT v32s16 = LLT::fixed_vector(32, 16);

  const LLT v16s32 = LLT::fixed_vector(16, 32);

  const LLT v8s64 = LLT::fixed_vector(8, 64);


  const LLT s8MaxVector = HasAVX512 ? v64s8 : HasAVX ? v32s8 : v16s8;

  const LLT s16MaxVector = HasAVX512 ? v32s16 : HasAVX ? v16s16 : v8s16;

  const LLT s32MaxVector = HasAVX512 ? v16s32 : HasAVX ? v8s32 : v4s32;

  const LLT s64MaxVector = HasAVX512 ? v8s64 : HasAVX ? v4s64 : v2s64;


  // todo: AVX512 bool vector predicate types


  // implicit/constants

  // 32/64-bits needs support for s64/s128 to handle cases:

  // s64 = EXTEND (G_IMPLICIT_DEF s32) -> s64 = G_IMPLICIT_DEF

  // s128 = EXTEND (G_IMPLICIT_DEF s32/s64) -> s128 = G_IMPLICIT_DEF

  getActionDefinitionsBuilder(

      {G_IMPLICIT_DEF, G_PHI, G_FREEZE, G_CONSTANT_FOLD_BARRIER})

      .legalFor({p0, s1, s8, s16, s32, s64})

      .legalFor(UseX87, {s80})

      .legalFor(Is64Bit, {s128})

      .legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})

      .legalFor(HasAVX, {v32s8, v16s16, v8s32, v4s64})

      .legalFor(HasAVX512, {v64s8, v32s16, v16s32, v8s64})

      .widenScalarOrEltToNextPow2(0, /*Min=*/8)

      .clampScalarOrElt(0, s8, sMaxScalar)

      .moreElementsToNextPow2(0)

      .clampMinNumElements(0, s8, 16)

      .clampMinNumElements(0, s16, 8)

      .clampMinNumElements(0, s32, 4)

      .clampMinNumElements(0, s64, 2)

      .clampMaxNumElements(0, s8, HasAVX512 ? 64 : (HasAVX ? 32 : 16))

      .clampMaxNumElements(0, s16, HasAVX512 ? 32 : (HasAVX ? 16 : 8))

      .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX ? 8 : 4))

      .clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX ? 4 : 2))

      .clampMaxNumElements(0, p0,

                           Is64Bit ? s64MaxVector.getNumElements()

                                   : s32MaxVector.getNumElements())

      .scalarizeIf(scalarOrEltWiderThan(0, 64), 0);


  getActionDefinitionsBuilder(G_CONSTANT)

      .legalFor({p0, s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .widenScalarToNextPow2(0, /*Min=*/8)

      .clampScalar(0, s8, sMaxScalar);


  getActionDefinitionsBuilder({G_LROUND,  G_LLROUND, G_FCOS,  G_FCOSH,  G_FACOS,

                               G_FSIN,    G_FSINH,   G_FASIN, G_FTAN,   G_FTANH,

                               G_FATAN,   G_FATAN2,  G_FPOW,  G_FEXP,   G_FEXP2,

                               G_FEXP10,  G_FLOG,    G_FLOG2, G_FLOG10, G_FPOWI,

                               G_FSINCOS, G_FCEIL,   G_FFLOOR})

      .libcall();


  getActionDefinitionsBuilder(G_FSQRT)

      .legalFor(HasSSE1 || UseX87, {s32})

      .legalFor(HasSSE2 || UseX87, {s64})

      .legalFor(UseX87, {s80});


  getActionDefinitionsBuilder({G_GET_ROUNDING, G_SET_ROUNDING})

      .customFor({s32});


  // merge/unmerge

  for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {

    unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;

    unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;

    getActionDefinitionsBuilder(Op)

        .widenScalarToNextPow2(LitTyIdx, /*Min=*/8)

        .widenScalarToNextPow2(BigTyIdx, /*Min=*/16)

        .minScalar(LitTyIdx, s8)

        .minScalar(BigTyIdx, s32)

        .legalIf([=](const LegalityQuery &Q) {

          switch (Q.Types[BigTyIdx].getSizeInBits()) {

          case 16:

          case 32:

          case 64:

          case 128:

          case 256:

          case 512:

            break;

          default:

            return false;

          }

          switch (Q.Types[LitTyIdx].getSizeInBits()) {

          case 8:

          case 16:

          case 32:

          case 64:

          case 128:

          case 256:

            return true;

          default:

            return false;

          }

        });

  }


  getActionDefinitionsBuilder({G_UMIN, G_UMAX, G_SMIN, G_SMAX})

      .widenScalarToNextPow2(0, /*Min=*/32)

      .lower();


  // integer addition/subtraction

  getActionDefinitionsBuilder({G_ADD, G_SUB})

      .legalFor({s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})

      .legalFor(HasAVX2, {v32s8, v16s16, v8s32, v4s64})

      .legalFor(HasAVX512, {v16s32, v8s64})

      .legalFor(HasBWI, {v64s8, v32s16})

      .clampMinNumElements(0, s8, 16)

      .clampMinNumElements(0, s16, 8)

      .clampMinNumElements(0, s32, 4)

      .clampMinNumElements(0, s64, 2)

      .clampMaxNumElements(0, s8, HasBWI ? 64 : (HasAVX2 ? 32 : 16))

      .clampMaxNumElements(0, s16, HasBWI ? 32 : (HasAVX2 ? 16 : 8))

      .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))

      .clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX2 ? 4 : 2))

      .widenScalarToNextPow2(0, /*Min=*/32)

      .clampScalar(0, s8, sMaxScalar)

      .scalarize(0);


  getActionDefinitionsBuilder({G_UADDE, G_UADDO, G_USUBE, G_USUBO})

      .legalFor({{s8, s1}, {s16, s1}, {s32, s1}})

      .legalFor(Is64Bit, {{s64, s1}})

      .widenScalarToNextPow2(0, /*Min=*/32)

      .clampScalar(0, s8, sMaxScalar)

      .clampScalar(1, s1, s1)

      .scalarize(0);


  // integer multiply

  getActionDefinitionsBuilder(G_MUL)

      .legalFor({s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .legalFor(HasSSE2, {v8s16})

      .legalFor(HasSSE41, {v4s32})

      .legalFor(HasAVX2, {v16s16, v8s32})

      .legalFor(HasAVX512, {v16s32})

      .legalFor(HasDQI, {v8s64})

      .legalFor(HasDQI && HasVLX, {v2s64, v4s64})

      .legalFor(HasBWI, {v32s16})

      .clampMinNumElements(0, s16, 8)

      .clampMinNumElements(0, s32, 4)

      .clampMinNumElements(0, s64, HasVLX ? 2 : 8)

      .clampMaxNumElements(0, s16, HasBWI ? 32 : (HasAVX2 ? 16 : 8))

      .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))

      .clampMaxNumElements(0, s64, 8)

      .widenScalarToNextPow2(0, /*Min=*/32)

      .clampScalar(0, s8, sMaxScalar)

      .scalarize(0);


  getActionDefinitionsBuilder({G_SMULH, G_UMULH})

      .legalFor({s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .widenScalarToNextPow2(0, /*Min=*/32)

      .clampScalar(0, s8, sMaxScalar)

      .scalarize(0);


  // integer divisions

  getActionDefinitionsBuilder({G_SDIV, G_SREM, G_UDIV, G_UREM})

      .legalFor({s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .libcallFor({s64})

      .clampScalar(0, s8, sMaxScalar);


  // integer shifts

  getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})

      .legalFor({{s8, s8}, {s16, s8}, {s32, s8}})

      .legalFor(Is64Bit, {{s64, s8}})

      .clampScalar(0, s8, sMaxScalar)

      .clampScalar(1, s8, s8);


  // integer logic

  getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})

      .legalFor({s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})

      .legalFor(HasAVX, {v32s8, v16s16, v8s32, v4s64})

      .legalFor(HasAVX512, {v64s8, v32s16, v16s32, v8s64})

      .clampMinNumElements(0, s8, 16)

      .clampMinNumElements(0, s16, 8)

      .clampMinNumElements(0, s32, 4)

      .clampMinNumElements(0, s64, 2)

      .clampMaxNumElements(0, s8, HasAVX512 ? 64 : (HasAVX ? 32 : 16))

      .clampMaxNumElements(0, s16, HasAVX512 ? 32 : (HasAVX ? 16 : 8))

      .clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX ? 8 : 4))

      .clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX ? 4 : 2))

      .widenScalarToNextPow2(0, /*Min=*/32)

      .clampScalar(0, s8, sMaxScalar)

      .scalarize(0);


  // integer comparison

  const std::initializer_list<LLT> IntTypes32 = {s8, s16, s32, p0};

  const std::initializer_list<LLT> IntTypes64 = {s8, s16, s32, s64, p0};


  getActionDefinitionsBuilder(G_ICMP)

      .legalForCartesianProduct({s8}, Is64Bit ? IntTypes64 : IntTypes32)

      .clampScalar(0, s8, s8)

      .clampScalar(1, s8, sMaxScalar);


  // bswap

  getActionDefinitionsBuilder(G_BSWAP)

      .legalFor({s32})

      .legalFor(Is64Bit, {s64})

      .widenScalarToNextPow2(0, /*Min=*/32)

      .clampScalar(0, s32, sMaxScalar);


  // popcount

  getActionDefinitionsBuilder(G_CTPOP)

      .legalFor(HasPOPCNT, {{s16, s16}, {s32, s32}})

      .legalFor(HasPOPCNT && Is64Bit, {{s64, s64}})

      .widenScalarToNextPow2(1, /*Min=*/16)

      .clampScalar(1, s16, sMaxScalar)

      .scalarSameSizeAs(0, 1);


  // count leading zeros (LZCNT)

  getActionDefinitionsBuilder(G_CTLZ)

      .legalFor(HasLZCNT, {{s16, s16}, {s32, s32}})

      .legalFor(HasLZCNT && Is64Bit, {{s64, s64}})

      .widenScalarToNextPow2(1, /*Min=*/16)

      .clampScalar(1, s16, sMaxScalar)

      .scalarSameSizeAs(0, 1);


  // count trailing zeros

  getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF)

      .legalFor({{s16, s16}, {s32, s32}})

      .legalFor(Is64Bit, {{s64, s64}})

      .widenScalarToNextPow2(1, /*Min=*/16)

      .clampScalar(1, s16, sMaxScalar)

      .scalarSameSizeAs(0, 1);


  getActionDefinitionsBuilder(G_CTTZ)

      .legalFor(HasBMI, {{s16, s16}, {s32, s32}})

      .legalFor(HasBMI && Is64Bit, {{s64, s64}})

      .widenScalarToNextPow2(1, /*Min=*/16)

      .clampScalar(1, s16, sMaxScalar)

      .scalarSameSizeAs(0, 1);


  getActionDefinitionsBuilder(G_BRCOND).legalFor({s1});


  // pointer handling

  const std::initializer_list<LLT> PtrTypes32 = {s1, s8, s16, s32};

  const std::initializer_list<LLT> PtrTypes64 = {s1, s8, s16, s32, s64};


  getActionDefinitionsBuilder(G_PTRTOINT)

      .legalForCartesianProduct(Is64Bit ? PtrTypes64 : PtrTypes32, {p0})

      .maxScalar(0, sMaxScalar)

      .widenScalarToNextPow2(0, /*Min*/ 8);


  getActionDefinitionsBuilder(G_INTTOPTR).legalFor({{p0, sMaxScalar}});


  getActionDefinitionsBuilder(G_CONSTANT_POOL).legalFor({p0});


  getActionDefinitionsBuilder(G_PTR_ADD)

      .legalFor({{p0, s32}})

      .legalFor(Is64Bit, {{p0, s64}})

      .widenScalarToNextPow2(1, /*Min*/ 32)

      .clampScalar(1, s32, sMaxScalar);


  getActionDefinitionsBuilder({G_FRAME_INDEX, G_GLOBAL_VALUE}).legalFor({p0});


  // load/store: add more corner cases

  for (unsigned Op : {G_LOAD, G_STORE}) {

    auto &Action = getActionDefinitionsBuilder(Op);

    Action.legalForTypesWithMemDesc({{s8, p0, s8, 1},

                                     {s16, p0, s16, 1},

                                     {s32, p0, s32, 1},

                                     {s80, p0, s80, 1},

                                     {p0, p0, p0, 1},

                                     {v4s8, p0, v4s8, 1}});

    if (Is64Bit)

      Action.legalForTypesWithMemDesc(

          {{s64, p0, s64, 1}, {v2s32, p0, v2s32, 1}});


    if (HasSSE1)

      Action.legalForTypesWithMemDesc({{v4s32, p0, v4s32, 1}});

    if (HasSSE2)

      Action.legalForTypesWithMemDesc({{v16s8, p0, v16s8, 1},

                                       {v8s16, p0, v8s16, 1},

                                       {v2s64, p0, v2s64, 1},

                                       {v2p0, p0, v2p0, 1}});

    if (HasAVX)

      Action.legalForTypesWithMemDesc({{v32s8, p0, v32s8, 1},

                                       {v16s16, p0, v16s16, 1},

                                       {v8s32, p0, v8s32, 1},

                                       {v4s64, p0, v4s64, 1},

                                       {v4p0, p0, v4p0, 1}});

    if (HasAVX512)

      Action.legalForTypesWithMemDesc({{v64s8, p0, v64s8, 1},

                                       {v32s16, p0, v32s16, 1},

                                       {v16s32, p0, v16s32, 1},

                                       {v8s64, p0, v8s64, 1}});


    // X86 supports extending loads but not stores for GPRs

    if (Op == G_LOAD) {

      Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},

                                       {s16, p0, s8, 1},

                                       {s32, p0, s8, 1},

                                       {s32, p0, s16, 1}});

      if (Is64Bit)

        Action.legalForTypesWithMemDesc(

            {{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});

    } else {

      Action.customIf([=](const LegalityQuery &Query) {

        return Query.Types[0] != Query.MMODescrs[0].MemoryTy;

      });

    }

    Action.widenScalarToNextPow2(0, /*Min=*/8)

        .clampScalar(0, s8, sMaxScalar)

        .scalarize(0);

  }


  for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {

    auto &Action = getActionDefinitionsBuilder(Op);

    Action.legalForTypesWithMemDesc(

        {{s16, p0, s8, 1}, {s32, p0, s8, 1}, {s32, p0, s16, 1}});

    if (Is64Bit)

      Action.legalForTypesWithMemDesc(

          {{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});

    // TODO - SSE41/AVX2/AVX512F/AVX512BW vector extensions

  }


  // sext, zext, and anyext

  getActionDefinitionsBuilder(G_ANYEXT)

      .legalFor({s8, s16, s32, s128})

      .legalFor(Is64Bit, {s64})

      .widenScalarToNextPow2(0, /*Min=*/8)

      .clampScalar(0, s8, sMaxScalar)

      .widenScalarToNextPow2(1, /*Min=*/8)

      .clampScalar(1, s8, sMaxScalar)

      .scalarize(0);


  getActionDefinitionsBuilder({G_SEXT, G_ZEXT})

      .legalFor({s8, s16, s32})

      .legalFor(Is64Bit, {s64})

      .widenScalarToNextPow2(0, /*Min=*/8)

      .clampScalar(0, s8, sMaxScalar)

      .widenScalarToNextPow2(1, /*Min=*/8)

      .clampScalar(1, s8, sMaxScalar)

      .scalarize(0);


  getActionDefinitionsBuilder(G_SEXT_INREG).lower();


  // fp constants

  getActionDefinitionsBuilder(G_FCONSTANT)

      .legalFor({s32, s64})

      .legalFor(UseX87, {s80});


  // fp arithmetic

  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV})

      .legalFor({s32, s64})

      .legalFor(HasSSE1, {v4s32})

      .legalFor(HasSSE2, {v2s64})

      .legalFor(HasAVX, {v8s32, v4s64})

      .legalFor(HasAVX512, {v16s32, v8s64})

      .legalFor(UseX87, {s80});


  getActionDefinitionsBuilder(G_FABS)

      .legalFor(UseX87, {s80})

      .legalFor(UseX87 && !Is64Bit, {s64})

      .lower();


  // fp comparison

  getActionDefinitionsBuilder(G_FCMP)

      .legalFor(HasSSE1 || UseX87, {s8, s32})

      .legalFor(HasSSE2 || UseX87, {s8, s64})

      .legalFor(UseX87, {s8, s80})

      .clampScalar(0, s8, s8)

      .clampScalar(1, s32, HasSSE2 ? s64 : s32)

      .widenScalarToNextPow2(1);


  // fp conversions

  getActionDefinitionsBuilder(G_FPEXT)

      .legalFor(HasSSE2, {{s64, s32}})

      .legalFor(HasAVX, {{v4s64, v4s32}})

      .legalFor(HasAVX512, {{v8s64, v8s32}});


  getActionDefinitionsBuilder(G_FPTRUNC)

      .legalFor(HasSSE2, {{s32, s64}})

      .legalFor(HasAVX, {{v4s32, v4s64}})

      .legalFor(HasAVX512, {{v8s32, v8s64}});


  getActionDefinitionsBuilder(G_SITOFP)

      .legalFor(HasSSE1, {{s32, s32}})

      .legalFor(HasSSE1 && Is64Bit, {{s32, s64}})

      .legalFor(HasSSE2, {{s64, s32}})

      .legalFor(HasSSE2 && Is64Bit, {{s64, s64}})

      .clampScalar(1, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)

      .widenScalarToNextPow2(1)

      .customForCartesianProduct(UseX87, {s32, s64, s80}, {s16, s32, s64})

      .clampScalar(0, s32, HasSSE2 ? s64 : s32)

      .widenScalarToNextPow2(0);


  getActionDefinitionsBuilder(G_FPTOSI)

      .legalFor(HasSSE1, {{s32, s32}})

      .legalFor(HasSSE1 && Is64Bit, {{s64, s32}})

      .legalFor(HasSSE2, {{s32, s64}})

      .legalFor(HasSSE2 && Is64Bit, {{s64, s64}})

      .clampScalar(0, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)

      .widenScalarToNextPow2(0)

      .customForCartesianProduct(UseX87, {s16, s32, s64}, {s32, s64, s80})

      .clampScalar(1, s32, HasSSE2 ? s64 : s32)

      .widenScalarToNextPow2(1);


  // For G_UITOFP and G_FPTOUI without AVX512, we have to custom legalize types

  // <= s32 manually. Otherwise, in custom handler there is no way to

  // understand whether s32 is an original type and we need to promote it to

  // s64 or s32 is obtained after widening and we shouldn't widen it to s64.

  //

  // For AVX512 we simply widen types as there is direct mapping from opcodes

  // to asm instructions.

  getActionDefinitionsBuilder(G_UITOFP)

      .legalFor(HasAVX512, {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})

      .customIf([=](const LegalityQuery &Query) {

        return !HasAVX512 &&

               ((HasSSE1 && typeIs(0, s32)(Query)) ||

                (HasSSE2 && typeIs(0, s64)(Query))) &&

               scalarNarrowerThan(1, Is64Bit ? 64 : 32)(Query);

      })

      .lowerIf([=](const LegalityQuery &Query) {

        // Lower conversions from s64

        return !HasAVX512 &&

               ((HasSSE1 && typeIs(0, s32)(Query)) ||

                (HasSSE2 && typeIs(0, s64)(Query))) &&

               (Is64Bit && typeIs(1, s64)(Query));

      })

      .clampScalar(0, s32, HasSSE2 ? s64 : s32)

      .widenScalarToNextPow2(0)

      .clampScalar(1, s32, sMaxScalar)

      .widenScalarToNextPow2(1);


  getActionDefinitionsBuilder(G_FPTOUI)

      .legalFor(HasAVX512, {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})

      .customIf([=](const LegalityQuery &Query) {

        return !HasAVX512 &&

               ((HasSSE1 && typeIs(1, s32)(Query)) ||

                (HasSSE2 && typeIs(1, s64)(Query))) &&

               scalarNarrowerThan(0, Is64Bit ? 64 : 32)(Query);

      })

      // TODO: replace with customized legalization using

      // specifics of cvttsd2si. The selection of this node requires

      // a vector type. Either G_SCALAR_TO_VECTOR is needed or more advanced

      // support of G_BUILD_VECTOR/G_INSERT_VECTOR_ELT is required beforehand.

      .lowerIf([=](const LegalityQuery &Query) {

        return !HasAVX512 &&

               ((HasSSE1 && typeIs(1, s32)(Query)) ||

                (HasSSE2 && typeIs(1, s64)(Query))) &&

               (Is64Bit && typeIs(0, s64)(Query));

      })

      .clampScalar(0, s32, sMaxScalar)

      .widenScalarToNextPow2(0)

      .clampScalar(1, s32, HasSSE2 ? s64 : s32)

      .widenScalarToNextPow2(1);


  // vector ops

  getActionDefinitionsBuilder(G_BUILD_VECTOR)

      .customIf([=](const LegalityQuery &Query) {

        return (HasSSE1 && typeInSet(0, {v4s32})(Query)) ||

               (HasSSE2 && typeInSet(0, {v2s64, v8s16, v16s8})(Query)) ||

               (HasAVX && typeInSet(0, {v4s64, v8s32, v16s16, v32s8})(Query)) ||

               (HasAVX512 && typeInSet(0, {v8s64, v16s32, v32s16, v64s8}));

      })

      .clampNumElements(0, v16s8, s8MaxVector)

      .clampNumElements(0, v8s16, s16MaxVector)

      .clampNumElements(0, v4s32, s32MaxVector)

      .clampNumElements(0, v2s64, s64MaxVector)

      .moreElementsToNextPow2(0);


  getActionDefinitionsBuilder({G_EXTRACT, G_INSERT})

      .legalIf([=](const LegalityQuery &Query) {

        unsigned SubIdx = Query.Opcode == G_EXTRACT ? 0 : 1;

        unsigned FullIdx = Query.Opcode == G_EXTRACT ? 1 : 0;

        return (HasAVX && typePairInSet(SubIdx, FullIdx,

                                        {{v16s8, v32s8},

                                         {v8s16, v16s16},

                                         {v4s32, v8s32},

                                         {v2s64, v4s64}})(Query)) ||

               (HasAVX512 && typePairInSet(SubIdx, FullIdx,

                                           {{v16s8, v64s8},

                                            {v32s8, v64s8},

                                            {v8s16, v32s16},

                                            {v16s16, v32s16},

                                            {v4s32, v16s32},

                                            {v8s32, v16s32},

                                            {v2s64, v8s64},

                                            {v4s64, v8s64}})(Query));

      });


  // todo: only permit dst types up to max legal vector register size?

  getActionDefinitionsBuilder(G_CONCAT_VECTORS)

      .legalFor(

          HasSSE1,

          {{v32s8, v16s8}, {v16s16, v8s16}, {v8s32, v4s32}, {v4s64, v2s64}})

      .legalFor(HasAVX, {{v64s8, v16s8},

                         {v64s8, v32s8},

                         {v32s16, v8s16},

                         {v32s16, v16s16},

                         {v16s32, v4s32},

                         {v16s32, v8s32},

                         {v8s64, v2s64},

                         {v8s64, v4s64}});


  // todo: vectors and address spaces

  getActionDefinitionsBuilder(G_SELECT)

      .legalFor({{s8, s32}, {s16, s32}, {s32, s32}, {s64, s32}, {p0, s32}})

      .widenScalarToNextPow2(0, /*Min=*/8)

      .clampScalar(0, HasCMOV ? s16 : s8, sMaxScalar)

      .clampScalar(1, s32, s32);


  // memory intrinsics

  getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();


  getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})

      .lower();


  // fp intrinsics

  getActionDefinitionsBuilder({G_INTRINSIC_ROUNDEVEN, G_INTRINSIC_TRUNC})

      .scalarize(0)

      .minScalar(0, LLT::scalar(32))

      .libcall();


  getLegacyLegalizerInfo().computeTables();

  verify(*STI.getInstrInfo());

}


bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,

                                      LostDebugLocObserver &LocObserver) const {

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();

  switch (MI.getOpcode()) {

  default:

    // No idea what to do.

    return false;

  case TargetOpcode::G_BUILD_VECTOR:

    return legalizeBuildVector(MI, MRI, Helper);

  case TargetOpcode::G_FPTOUI:

    return legalizeFPTOUI(MI, MRI, Helper);

  case TargetOpcode::G_UITOFP:

    return legalizeUITOFP(MI, MRI, Helper);

  case TargetOpcode::G_STORE:

    return legalizeNarrowingStore(MI, MRI, Helper);

  case TargetOpcode::G_SITOFP:

    return legalizeSITOFP(MI, MRI, Helper);

  case TargetOpcode::G_FPTOSI:

    return legalizeFPTOSI(MI, MRI, Helper);

  case TargetOpcode::G_GET_ROUNDING:

    return legalizeGETROUNDING(MI, MRI, Helper);

  case TargetOpcode::G_SET_ROUNDING:

    return legalizeSETROUNDING(MI, MRI, Helper);

  }

  llvm_unreachable("expected switch to return");

}


bool X86LegalizerInfo::legalizeSITOFP(MachineInstr &MI,

                                      MachineRegisterInfo &MRI,

                                      LegalizerHelper &Helper) const {

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  MachineFunction &MF = *MI.getMF();

  auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();


  assert((SrcTy.getSizeInBits() == 16 || SrcTy.getSizeInBits() == 32 ||

          SrcTy.getSizeInBits() == 64) &&

         "Unexpected source type for SITOFP in X87 mode.");


  TypeSize MemSize = SrcTy.getSizeInBytes();

  MachinePointerInfo PtrInfo;

  Align Alignmt = Helper.getStackTemporaryAlignment(SrcTy);

  auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);

  MachineMemOperand *StoreMMO = MF.getMachineMemOperand(

      PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));


  // Store the integer value on the FPU stack.

  MIRBuilder.buildStore(Src, SlotPointer, *StoreMMO);


  MachineMemOperand *LoadMMO = MF.getMachineMemOperand(

      PtrInfo, MachineMemOperand::MOLoad, MemSize, Align(MemSize));

  MIRBuilder.buildInstr(X86::G_FILD)

      .addDef(Dst)

      .addUse(SlotPointer.getReg(0))

      .addMemOperand(LoadMMO);


  MI.eraseFromParent();

  return true;

}


bool X86LegalizerInfo::legalizeFPTOSI(MachineInstr &MI,

                                      MachineRegisterInfo &MRI,

                                      LegalizerHelper &Helper) const {

  MachineFunction &MF = *MI.getMF();

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();


  TypeSize MemSize = DstTy.getSizeInBytes();

  MachinePointerInfo PtrInfo;

  Align Alignmt = Helper.getStackTemporaryAlignment(DstTy);

  auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);

  MachineMemOperand *StoreMMO = MF.getMachineMemOperand(

      PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));


  MIRBuilder.buildInstr(X86::G_FIST)

      .addUse(Src)

      .addUse(SlotPointer.getReg(0))

      .addMemOperand(StoreMMO);


  MIRBuilder.buildLoad(Dst, SlotPointer, PtrInfo, Align(MemSize));

  MI.eraseFromParent();

  return true;

}


bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,

                                           MachineRegisterInfo &MRI,

                                           LegalizerHelper &Helper) const {

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  const auto &BuildVector = cast<GBuildVector>(MI);

  Register Dst = BuildVector.getReg(0);

  LLT DstTy = MRI.getType(Dst);

  MachineFunction &MF = MIRBuilder.getMF();

  LLVMContext &Ctx = MF.getFunction().getContext();

  uint64_t DstTySize = DstTy.getScalarSizeInBits();


  SmallVector<Constant *, 4> CstIdxs;

  for (unsigned i = 0; i < BuildVector.getNumSources(); ++i) {

    Register Source = BuildVector.getSourceReg(i);


    auto ValueAndReg = getIConstantVRegValWithLookThrough(Source, MRI);

    if (ValueAndReg) {

      CstIdxs.emplace_back(ConstantInt::get(Ctx, ValueAndReg->Value));

      continue;

    }


    auto FPValueAndReg = getFConstantVRegValWithLookThrough(Source, MRI);

    if (FPValueAndReg) {

      CstIdxs.emplace_back(ConstantFP::get(Ctx, FPValueAndReg->Value));

      continue;

    }


    if (getOpcodeDef<GImplicitDef>(Source, MRI)) {

      CstIdxs.emplace_back(UndefValue::get(Type::getIntNTy(Ctx, DstTySize)));

      continue;

    }

    return false;

  }


  Constant *ConstVal = ConstantVector::get(CstIdxs);


  const DataLayout &DL = MIRBuilder.getDataLayout();

  unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();

  Align Alignment(DL.getABITypeAlign(ConstVal->getType()));

  auto Addr = MIRBuilder.buildConstantPool(

      LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)),

      MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment));

  MachineMemOperand *MMO =

      MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),

                              MachineMemOperand::MOLoad, DstTy, Alignment);


  MIRBuilder.buildLoad(Dst, Addr, *MMO);

  MI.eraseFromParent();

  return true;

}


bool X86LegalizerInfo::legalizeFPTOUI(MachineInstr &MI,

                                      MachineRegisterInfo &MRI,

                                      LegalizerHelper &Helper) const {

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();

  unsigned DstSizeInBits = DstTy.getScalarSizeInBits();

  const LLT s32 = LLT::scalar(32);

  const LLT s64 = LLT::scalar(64);


  // Simply reuse FPTOSI when it is possible to widen the type

  if (DstSizeInBits <= 32) {

    auto Casted = MIRBuilder.buildFPTOSI(DstTy == s32 ? s64 : s32, Src);

    MIRBuilder.buildTrunc(Dst, Casted);

    MI.eraseFromParent();

    return true;

  }


  return false;

}


bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,

                                      MachineRegisterInfo &MRI,

                                      LegalizerHelper &Helper) const {

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();

  const LLT s32 = LLT::scalar(32);

  const LLT s64 = LLT::scalar(64);


  // Simply reuse SITOFP when it is possible to widen the type

  if (SrcTy.getSizeInBits() <= 32) {

    auto Ext = MIRBuilder.buildZExt(SrcTy == s32 ? s64 : s32, Src);

    MIRBuilder.buildSITOFP(Dst, Ext);

    MI.eraseFromParent();

    return true;

  }


  return false;

}


bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,

                                              MachineRegisterInfo &MRI,

                                              LegalizerHelper &Helper) const {

  auto &Store = cast<GStore>(MI);

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  MachineMemOperand &MMO = **Store.memoperands_begin();

  MachineFunction &MF = MIRBuilder.getMF();

  LLT ValTy = MRI.getType(Store.getValueReg());

  auto *NewMMO = MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), ValTy);


  Helper.Observer.changingInstr(Store);

  Store.setMemRefs(MF, {NewMMO});

  Helper.Observer.changedInstr(Store);

  return true;

}


bool X86LegalizerInfo::legalizeGETROUNDING(MachineInstr &MI,

                                           MachineRegisterInfo &MRI,

                                           LegalizerHelper &Helper) const {

  /*

   The rounding mode is in bits 11:10 of FPSR, and has the following

   settings:

     00 Round to nearest

     01 Round to -inf

     10 Round to +inf

     11 Round to 0


  GET_ROUNDING, on the other hand, expects the following:

    -1 Undefined

     0 Round to 0

     1 Round to nearest

     2 Round to +inf

     3 Round to -inf


  To perform the conversion, we use a packed lookup table of the four 2-bit

  values that we can index by FPSP[11:10]

    0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]


    (0x2d >> ((FPSR >> 9) & 6)) & 3

  */


  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  MachineFunction &MF = MIRBuilder.getMF();

  Register Dst = MI.getOperand(0).getReg();

  LLT DstTy = MRI.getType(Dst);

  const LLT s8 = LLT::scalar(8);

  const LLT s16 = LLT::scalar(16);

  const LLT s32 = LLT::scalar(32);


  // Save FP Control Word to stack slot

  int MemSize = 2;

  Align Alignment = Align(2);

  MachinePointerInfo PtrInfo;

  auto StackTemp = Helper.createStackTemporary(TypeSize::getFixed(MemSize),

                                               Alignment, PtrInfo);

  Register StackPtr = StackTemp.getReg(0);


  auto StoreMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,

                                          MemSize, Alignment);


  // Store FP Control Word to stack slot using G_FNSTCW16

  MIRBuilder.buildInstr(X86::G_FNSTCW16)

      .addUse(StackPtr)

      .addMemOperand(StoreMMO);


  // Load FP Control Word from stack slot

  auto LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,

                                         MemSize, Alignment);


  auto CWD32 =

      MIRBuilder.buildZExt(s32, MIRBuilder.buildLoad(s16, StackPtr, *LoadMMO));

  auto Shifted8 = MIRBuilder.buildTrunc(

      s8, MIRBuilder.buildLShr(s32, CWD32, MIRBuilder.buildConstant(s8, 9)));

  auto Masked32 = MIRBuilder.buildZExt(

      s32, MIRBuilder.buildAnd(s8, Shifted8, MIRBuilder.buildConstant(s8, 6)));


  // LUT is a packed lookup table (0x2d) used to map the 2-bit x87 FPU rounding

  // mode (from bits 11:10 of the control word) to the values expected by

  // GET_ROUNDING. The mapping is performed by shifting LUT right by the

  // extracted rounding mode and masking the result with 3 to obtain the final

  auto LUT = MIRBuilder.buildConstant(s32, 0x2d);

  auto LUTShifted = MIRBuilder.buildLShr(s32, LUT, Masked32);

  auto RetVal =

      MIRBuilder.buildAnd(s32, LUTShifted, MIRBuilder.buildConstant(s32, 3));

  auto RetValTrunc = MIRBuilder.buildZExtOrTrunc(DstTy, RetVal);


  MIRBuilder.buildCopy(Dst, RetValTrunc);


  MI.eraseFromParent();

  return true;

}


bool X86LegalizerInfo::legalizeSETROUNDING(MachineInstr &MI,

                                           MachineRegisterInfo &MRI,

                                           LegalizerHelper &Helper) const {

  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;

  MachineFunction &MF = MIRBuilder.getMF();

  Register Src = MI.getOperand(0).getReg();

  const LLT s8 = LLT::scalar(8);

  const LLT s16 = LLT::scalar(16);

  const LLT s32 = LLT::scalar(32);


  // Allocate stack slot for control word and MXCSR (4 bytes).

  int MemSize = 4;

  Align Alignment = Align(4);

  MachinePointerInfo PtrInfo;

  auto StackTemp = Helper.createStackTemporary(TypeSize::getFixed(MemSize),

                                               Alignment, PtrInfo);

  Register StackPtr = StackTemp.getReg(0);


  auto StoreMMO =

      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 2, Align(2));

  MIRBuilder.buildInstr(X86::G_FNSTCW16)

      .addUse(StackPtr)

      .addMemOperand(StoreMMO);


  auto LoadMMO =

      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 2, Align(2));

  auto CWD16 = MIRBuilder.buildLoad(s16, StackPtr, *LoadMMO);


  // Clear RM field (bits 11:10)

  auto ClearedCWD =

      MIRBuilder.buildAnd(s16, CWD16, MIRBuilder.buildConstant(s16, 0xf3ff));


  // Check if Src is a constant

  auto *SrcDef = MRI.getVRegDef(Src);

  Register RMBits;

  Register MXCSRRMBits;


  if (SrcDef && SrcDef->getOpcode() == TargetOpcode::G_CONSTANT) {

    uint64_t RM = getIConstantFromReg(Src, MRI).getZExtValue();

    int FieldVal = X86::getRoundingModeX86(RM);


    if (FieldVal == X86::rmInvalid) {

      FieldVal = X86::rmToNearest;

      LLVMContext &C = MF.getFunction().getContext();

      C.diagnose(DiagnosticInfoUnsupported(

          MF.getFunction(), "rounding mode is not supported by X86 hardware",

          DiagnosticLocation(MI.getDebugLoc()), DS_Error));

      return false;

    }


    FieldVal = FieldVal << 3;

    RMBits = MIRBuilder.buildConstant(s16, FieldVal).getReg(0);

    MXCSRRMBits = MIRBuilder.buildConstant(s32, FieldVal).getReg(0);

  } else {

    // Convert Src (rounding mode) to bits for control word

    // (0xc9 << (2 * Src + 4)) & 0xc00

    auto Src32 = MIRBuilder.buildZExtOrTrunc(s32, Src);

    auto ShiftAmt = MIRBuilder.buildAdd(

        s32, MIRBuilder.buildShl(s32, Src32, MIRBuilder.buildConstant(s32, 1)),

        MIRBuilder.buildConstant(s32, 4));

    auto ShiftAmt8 = MIRBuilder.buildTrunc(s8, ShiftAmt);

    auto Shifted = MIRBuilder.buildShl(s16, MIRBuilder.buildConstant(s16, 0xc9),

                                       ShiftAmt8);

    RMBits =

        MIRBuilder.buildAnd(s16, Shifted, MIRBuilder.buildConstant(s16, 0xc00))

            .getReg(0);


    // For non-constant case, we still need to compute MXCSR bits dynamically

    auto RMBits32 = MIRBuilder.buildZExt(s32, RMBits);

    MXCSRRMBits =

        MIRBuilder.buildShl(s32, RMBits32, MIRBuilder.buildConstant(s32, 3))

            .getReg(0);

  }

  // Update rounding mode bits

  auto NewCWD =

      MIRBuilder.buildOr(s16, ClearedCWD, RMBits, MachineInstr::Disjoint);


  // Store new FP Control Word to stack

  auto StoreNewMMO =

      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 2, Align(2));

  MIRBuilder.buildStore(NewCWD, StackPtr, *StoreNewMMO);


  // Load FP control word from the slot using G_FLDCW16

  auto LoadNewMMO =

      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 2, Align(2));

  MIRBuilder.buildInstr(X86::G_FLDCW16)

      .addUse(StackPtr)

      .addMemOperand(LoadNewMMO);


  if (Subtarget.hasSSE1()) {

    // Store MXCSR to stack (use STMXCSR)

    auto StoreMXCSRMMO = MF.getMachineMemOperand(

        PtrInfo, MachineMemOperand::MOStore, 4, Align(4));

    MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)

        .addIntrinsicID(Intrinsic::x86_sse_stmxcsr)

        .addUse(StackPtr)

        .addMemOperand(StoreMXCSRMMO);


    // Load MXCSR from stack

    auto LoadMXCSRMMO = MF.getMachineMemOperand(

        PtrInfo, MachineMemOperand::MOLoad, 4, Align(4));

    auto MXCSR = MIRBuilder.buildLoad(s32, StackPtr, *LoadMXCSRMMO);


    // Clear RM field (bits 14:13)

    auto ClearedMXCSR = MIRBuilder.buildAnd(

        s32, MXCSR, MIRBuilder.buildConstant(s32, 0xffff9fff));


    // Update rounding mode bits

    auto NewMXCSR = MIRBuilder.buildOr(s32, ClearedMXCSR, MXCSRRMBits);


    // Store new MXCSR to stack

    auto StoreNewMXCSRMMO = MF.getMachineMemOperand(

        PtrInfo, MachineMemOperand::MOStore, 4, Align(4));

    MIRBuilder.buildStore(NewMXCSR, StackPtr, *StoreNewMXCSRMMO);


    // Load MXCSR from stack (use LDMXCSR)

    auto LoadNewMXCSRMMO = MF.getMachineMemOperand(

        PtrInfo, MachineMemOperand::MOLoad, 4, Align(4));

    MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)

        .addIntrinsicID(Intrinsic::x86_sse_ldmxcsr)

        .addUse(StackPtr)

        .addMemOperand(LoadNewMXCSRMMO);

  }


  MI.eraseFromParent();

  return true;

}


bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,

                                         MachineInstr &MI) const {

  return true;

}


MRI
unsigned const MachineRegisterInfo * MRI
Definition AArch64AdvSIMDScalarPass.cpp:103

assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73

DerivedTypes.h

scalarize
static void scalarize(Instruction *I, SmallVectorImpl< Instruction * > &Replace)
Definition ExpandFp.cpp:942

GenericMachineInstrs.h
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...

libcall
@ libcall
Definition HWAddressSanitizer.cpp:239

MI
IRTranslator LLVM IR MI
Definition IRTranslator.cpp:110

Type.h

LegalizerHelper.h

MachineConstantPool.h
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...

MachineFrameInfo.h

MachineIRBuilder.h
This file declares the MachineIRBuilder class.

Register
Promote Memory to Register
Definition Mem2Reg.cpp:110

verify
ppc ctr loops verify
Definition PPCCTRLoopsVerify.cpp:71

TargetOpcodes.h

LUT
static const char LUT[]
Definition TypeSanitizer.cpp:195

ValueTypes.h

X86LegalizerInfo.h
This file declares the targeting of the Machinelegalizer class for X86.

X86Subtarget.h

X86TargetMachine.h

llvm::APInt::getZExtValue
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1540

llvm::ConstantVector::get
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
Definition Constants.cpp:1423

llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:359

llvm::GISelChangeObserver::changingInstr
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.

llvm::GISelChangeObserver::changedInstr
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.

llvm::LLT
Definition LowLevelType.h:40

llvm::LLT::getScalarSizeInBits
constexpr unsigned getScalarSizeInBits() const
Definition LowLevelType.h:265

llvm::LLT::scalar
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition LowLevelType.h:43

llvm::LLT::pointer
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition LowLevelType.h:58

llvm::LLT::fixed_vector
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition LowLevelType.h:101

llvm::LLVMContext::diagnose
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition LLVMContext.cpp:247

llvm::LegacyLegalizerInfo::computeTables
LLVM_ABI void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
Definition LegacyLegalizerInfo.cpp:105

llvm::LegalizeRuleSet::minScalar
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
Definition LegalizerInfo.h:1060

llvm::LegalizeRuleSet::legalFor
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
Definition LegalizerInfo.h:625

llvm::LegalizeRuleSet::scalarSameSizeAs
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
Definition LegalizerInfo.h:1171

llvm::LegalizeRuleSet::libcall
LegalizeRuleSet & libcall()
The instruction is emitted as a library call.
Definition LegalizerInfo.h:772

llvm::LegalizeRuleSet::clampMaxNumElements
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
Definition LegalizerInfo.h:1260

llvm::LegalizeRuleSet::clampMinNumElements
LegalizeRuleSet & clampMinNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MinElements)
Limit the number of elements in EltTy vectors to at least MinElements.
Definition LegalizerInfo.h:1222

llvm::LegalizeRuleSet::customForCartesianProduct
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
Definition LegalizerInfo.h:921

llvm::LegalizeRuleSet::moreElementsToNextPow2
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
Definition LegalizerInfo.h:1214

llvm::LegalizeRuleSet::lower
LegalizeRuleSet & lower()
The instruction is lowered.
Definition LegalizerInfo.h:708

llvm::LegalizeRuleSet::clampScalar
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
Definition LegalizerInfo.h:1126

llvm::LegalizeRuleSet::clampNumElements
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
Definition LegalizerInfo.h:1284

llvm::LegalizeRuleSet::customIf
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
Definition LegalizerInfo.h:894

llvm::LegalizeRuleSet::widenScalarToNextPow2
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
Definition LegalizerInfo.h:959

llvm::LegalizeRuleSet::scalarize
LegalizeRuleSet & scalarize(unsigned TypeIdx)
Definition LegalizerInfo.h:1005

llvm::LegalizeRuleSet::legalForCartesianProduct
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
Definition LegalizerInfo.h:674

llvm::LegalizeRuleSet::legalIf
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
Definition LegalizerInfo.h:618

llvm::LegalizerHelper
Definition LegalizerHelper.h:49

llvm::LegalizerHelper::createStackTemporary
LLVM_ABI MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo)
Create a stack temporary based on the size in bytes and the alignment.
Definition LegalizerHelper.cpp:4886

llvm::LegalizerHelper::Observer
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Definition LegalizerHelper.h:56

llvm::LegalizerHelper::MIRBuilder
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
Definition LegalizerHelper.h:53

llvm::LegalizerHelper::getStackTemporaryAlignment
LLVM_ABI Align getStackTemporaryAlignment(LLT Type, Align MinAlign=Align()) const
Return the alignment to use for a stack temporary object with the given type.
Definition LegalizerHelper.cpp:4875

llvm::LegalizerInfo::getActionDefinitionsBuilder
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
Definition LegalizerInfo.cpp:314

llvm::LegalizerInfo::getLegacyLegalizerInfo
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
Definition LegalizerInfo.h:1334

llvm::LostDebugLocObserver
Definition LostDebugLocObserver.h:20

llvm::MachineConstantPool::getConstantPoolIndex
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
Definition MachineFunction.cpp:1568

llvm::MachineFunction
Definition MachineFunction.h:286

llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition MachineFunction.cpp:536

llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition MachineFunction.h:733

llvm::MachineFunction::getConstantPool
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
Definition MachineFunction.h:794

llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition MachineIRBuilder.h:236

llvm::MachineIRBuilder::buildFPTOSI
MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FPTOSI Src0.
Definition MachineIRBuilder.h:2216

llvm::MachineIRBuilder::buildAdd
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
Definition MachineIRBuilder.h:1848

llvm::MachineIRBuilder::buildConstantPool
MachineInstrBuilder buildConstantPool(const DstOp &Res, unsigned Idx)
Build and insert Res = G_CONSTANT_POOL Idx.
Definition MachineIRBuilder.cpp:169

llvm::MachineIRBuilder::buildAnd
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
Definition MachineIRBuilder.h:1993

llvm::MachineIRBuilder::buildLShr
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Definition MachineIRBuilder.h:1970

llvm::MachineIRBuilder::buildZExt
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
Definition MachineIRBuilder.cpp:523

llvm::MachineIRBuilder::buildLoad
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
Definition MachineIRBuilder.h:1036

llvm::MachineIRBuilder::buildZExtOrTrunc
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
Definition MachineIRBuilder.cpp:594

llvm::MachineIRBuilder::buildShl
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Definition MachineIRBuilder.h:1964

llvm::MachineIRBuilder::buildStore
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
Definition MachineIRBuilder.cpp:486

llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition MachineIRBuilder.h:418

llvm::MachineIRBuilder::buildSITOFP
MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_SITOFP Src0.
Definition MachineIRBuilder.h:2206

llvm::MachineIRBuilder::getMF
MachineFunction & getMF()
Getter for the function we currently build.
Definition MachineIRBuilder.h:288

llvm::MachineIRBuilder::buildTrunc
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
Definition MachineIRBuilder.cpp:927

llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition MachineIRBuilder.h:310

llvm::MachineIRBuilder::buildOr
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
Definition MachineIRBuilder.h:2008

llvm::MachineIRBuilder::buildCopy
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
Definition MachineIRBuilder.cpp:328

llvm::MachineIRBuilder::getDataLayout
const DataLayout & getDataLayout() const
Definition MachineIRBuilder.h:298

llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition MachineIRBuilder.cpp:333

llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition MachineInstrBuilder.h:123

llvm::MachineInstrBuilder::addIntrinsicID
const MachineInstrBuilder & addIntrinsicID(Intrinsic::ID ID) const
Definition MachineInstrBuilder.h:280

llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition MachineInstrBuilder.h:152

llvm::MachineInstrBuilder::addMemOperand
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Definition MachineInstrBuilder.h:231

llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition MachineInstrBuilder.h:145

llvm::MachineInstr
Representation of each machine instruction.
Definition MachineInstr.h:72

llvm::MachineInstr::Disjoint
@ Disjoint
Definition MachineInstr.h:122

llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition MachineMemOperand.h:130

llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition MachineMemOperand.h:137

llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition MachineMemOperand.h:139

llvm::MachineMemOperand::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition MachineMemOperand.h:207

llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition MachineRegisterInfo.h:53

llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:937

llvm::TypeSize
Definition TypeSize.h:333

llvm::TypeSize::getFixed
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:344

llvm::Type::getIntNTy
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:301

llvm::UndefValue::get
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition Constants.cpp:1866

llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256

llvm::X86LegalizerInfo::legalizeCustom
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
Definition X86LegalizerInfo.cpp:599

llvm::X86LegalizerInfo::legalizeIntrinsic
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
Definition X86LegalizerInfo.cpp:993

llvm::X86LegalizerInfo::X86LegalizerInfo
X86LegalizerInfo(const X86Subtarget &STI, const X86TargetMachine &TM)
Definition X86LegalizerInfo.cpp:32

llvm::X86Subtarget
Definition X86Subtarget.h:52

llvm::X86Subtarget::getInstrInfo
const X86InstrInfo * getInstrInfo() const override
Definition X86Subtarget.h:122

llvm::X86TargetMachine
Definition X86TargetMachine.h:28

llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164

llvm::AArch64::RM
@ RM
Definition AArch64ISelLowering.h:34

llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Definition AMDGPUMetadata.h:183

llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34

llvm::ISD::Constant
@ Constant
Definition ISDOpcodes.h:86

llvm::LegalityPredicates
Definition LegalizerInfo.h:206

llvm::LegalityPredicates::scalarOrEltWiderThan
LLVM_ABI LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or a vector with an element type that's wider than the ...
Definition LegalityPredicates.cpp:158

llvm::LegalityPredicates::typeInSet
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
Definition LegalityPredicates.cpp:34

llvm::LegalityPredicates::typePairInSet
LLVM_ABI LegalityPredicate typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, std::initializer_list< std::pair< LLT, LLT > > TypesInit)
True iff the given types for the given pair of type indexes is one of the specified type pairs.
Definition LegalityPredicates.cpp:42

llvm::LegalityPredicates::typeIs
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Definition LegalityPredicates.cpp:28

llvm::LegalityPredicates::scalarNarrowerThan
LLVM_ABI LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's narrower than the given size.
Definition LegalityPredicates.cpp:118

llvm::LegalizeActions
Definition LegalizerInfo.h:44

llvm::MipsISD::Ext
@ Ext
Definition MipsISelLowering.h:157

llvm::SPII::Store
@ Store
Definition SparcInstrInfo.h:33

llvm::Sched::Source
@ Source
Definition TargetLowering.h:105

llvm::TargetOpcode
Invariant opcodes: All instruction sets have these as their low opcodes.
Definition TargetOpcodes.h:20

llvm::X86::rmToNearest
@ rmToNearest
Definition X86ISelLowering.h:1008

llvm::X86::rmInvalid
@ rmInvalid
Definition X86ISelLowering.h:1007

llvm::X86::getRoundingModeX86
int getRoundingModeX86(unsigned RM)
Convert LLVM rounding mode to X86 rounding mode.
Definition X86ISelLowering.cpp:5347

llvm::codeview::EncodedFramePtrReg::StackPtr
@ StackPtr
Definition CodeView.h:526

llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition AddressRanges.h:18

llvm::getOpcodeDef
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition Utils.cpp:651

llvm::getIConstantFromReg
LLVM_ABI const APInt & getIConstantFromReg(Register VReg, const MachineRegisterInfo &MRI)
VReg is defined by a G_CONSTANT, return the corresponding value.
Definition Utils.cpp:305

llvm::SmallVector
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
Definition SmallVector.h:1122

llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:22

llvm::getFConstantVRegValWithLookThrough
LLVM_ABI std::optional< FPValueAndVReg > getFConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_FCONSTANT returns it...
Definition Utils.cpp:447

llvm::DS_Error
@ DS_Error
Definition DiagnosticInfo.h:53

llvm::cast
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:560

llvm::getIConstantVRegValWithLookThrough
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:433

llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39

llvm::LegalityQuery
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
Definition LegalizerInfo.h:111

llvm::LegalityQuery::MMODescrs
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
Definition LegalizerInfo.h:133

llvm::LegalityQuery::Types
ArrayRef< LLT > Types
Definition LegalizerInfo.h:113

llvm::LegalityQuery::Opcode
unsigned Opcode
Definition LegalizerInfo.h:112

llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition MachineMemOperand.h:42

llvm::MachinePointerInfo::getConstantPool
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
Definition MachineOperand.cpp:1071