X86FixupVectorConstants.cpp

Go to the documentation of this file.

//===-- X86FixupVectorConstants.cpp - optimize constant generation  -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file examines all full size vector constant pool loads and attempts to
// replace them with smaller constant pool entries, including:
// * Converting AVX512 memory-fold instructions to their broadcast-fold form
// * TODO: Broadcasting of full width loads.
// * TODO: Sign/Zero extension of full width loads.
//
//===----------------------------------------------------------------------===//
 
#include "X86.h"
#include "X86InstrFoldTables.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineConstantPool.h"
 
using namespace llvm;
 
#define DEBUG_TYPE "x86-fixup-vector-constants"
 
STATISTIC(NumInstChanges, "Number of instructions changes");
 
namespace {
class X86FixupVectorConstantsPass : public MachineFunctionPass {
public:
  static char ID;
 
  X86FixupVectorConstantsPass() : MachineFunctionPass(ID) {}
 
  StringRef getPassName() const override {
    return "X86 Fixup Vector Constants";
  }
 
  bool runOnMachineFunction(MachineFunction &MF) override;
  bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
                          MachineInstr &MI);
 
  // This pass runs after regalloc and doesn't support VReg operands.
  MachineFunctionProperties getRequiredProperties() const override {
    return MachineFunctionProperties().set(
        MachineFunctionProperties::Property::NoVRegs);
  }
 
private:
  const X86InstrInfo *TII = nullptr;
  const X86Subtarget *ST = nullptr;
  const MCSchedModel *SM = nullptr;
};
} // end anonymous namespace
 
char X86FixupVectorConstantsPass::ID = 0;
 
INITIALIZE_PASS(X86FixupVectorConstantsPass, DEBUG_TYPE, DEBUG_TYPE, false, false)
 
FunctionPass *llvm::createX86FixupVectorConstants() {
  return new X86FixupVectorConstantsPass();
}
 
static const Constant *getConstantFromPool(const MachineInstr &MI,
                                           const MachineOperand &Op) {
  if (!Op.isCPI() || Op.getOffset() != 0)
    return nullptr;
 
  ArrayRef<MachineConstantPoolEntry> Constants =
      MI.getParent()->getParent()->getConstantPool()->getConstants();
  const MachineConstantPoolEntry &ConstantEntry = Constants[Op.getIndex()];
 
  // Bail if this is a machine constant pool entry, we won't be able to dig out
  // anything useful.
  if (ConstantEntry.isMachineConstantPoolEntry())
    return nullptr;
 
  return ConstantEntry.Val.ConstVal;
}
 
// Attempt to extract the full width of bits data from the constant.
static std::optional<APInt> extractConstantBits(const Constant *C) {
  unsigned NumBits = C->getType()->getPrimitiveSizeInBits();
 
  if (auto *CInt = dyn_cast<ConstantInt>(C))
    return CInt->getValue();
 
  if (auto *CFP = dyn_cast<ConstantFP>(C))
    return CFP->getValue().bitcastToAPInt();
 
  if (auto *CV = dyn_cast<ConstantVector>(C)) {
    if (auto *CVSplat = CV->getSplatValue(/*AllowUndefs*/ true)) {
      if (std::optional<APInt> Bits = extractConstantBits(CVSplat)) {
        assert((NumBits % Bits->getBitWidth()) == 0 && "Illegal splat");
        return APInt::getSplat(NumBits, *Bits);
      }
    }
  }
 
  if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
    bool IsInteger = CDS->getElementType()->isIntegerTy();
    bool IsFloat = CDS->getElementType()->isHalfTy() ||
                   CDS->getElementType()->isBFloatTy() ||
                   CDS->getElementType()->isFloatTy() ||
                   CDS->getElementType()->isDoubleTy();
    if (IsInteger || IsFloat) {
      APInt Bits = APInt::getZero(NumBits);
      unsigned EltBits = CDS->getElementType()->getPrimitiveSizeInBits();
      for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
        if (IsInteger)
          Bits.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
        else
          Bits.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
                          I * EltBits);
      }
      return Bits;
    }
  }
 
  return std::nullopt;
}
 
// Attempt to compute the splat width of bits data by normalizing the splat to
// remove undefs.
static std::optional<APInt> getSplatableConstant(const Constant *C,
                                                 unsigned SplatBitWidth) {
  const Type *Ty = C->getType();
  assert((Ty->getPrimitiveSizeInBits() % SplatBitWidth) == 0 &&
         "Illegal splat width");
 
  if (std::optional<APInt> Bits = extractConstantBits(C))
    if (Bits->isSplat(SplatBitWidth))
      return Bits->trunc(SplatBitWidth);
 
  // Detect general splats with undefs.
  // TODO: Do we need to handle NumEltsBits > SplatBitWidth splitting?
  if (auto *CV = dyn_cast<ConstantVector>(C)) {
    unsigned NumOps = CV->getNumOperands();
    unsigned NumEltsBits = Ty->getScalarSizeInBits();
    unsigned NumScaleOps = SplatBitWidth / NumEltsBits;
    if ((SplatBitWidth % NumEltsBits) == 0) {
      // Collect the elements and ensure that within the repeated splat sequence
      // they either match or are undef.
      SmallVector<Constant *, 16> Sequence(NumScaleOps, nullptr);
      for (unsigned Idx = 0; Idx != NumOps; ++Idx) {
        if (Constant *Elt = CV->getAggregateElement(Idx)) {
          if (isa<UndefValue>(Elt))
            continue;
          unsigned SplatIdx = Idx % NumScaleOps;
          if (!Sequence[SplatIdx] || Sequence[SplatIdx] == Elt) {
            Sequence[SplatIdx] = Elt;
            continue;
          }
        }
        return std::nullopt;
      }
      // Extract the constant bits forming the splat and insert into the bits
      // data, leave undef as zero.
      APInt SplatBits = APInt::getZero(SplatBitWidth);
      for (unsigned I = 0; I != NumScaleOps; ++I) {
        if (!Sequence[I])
          continue;
        if (std::optional<APInt> Bits = extractConstantBits(Sequence[I])) {
          SplatBits.insertBits(*Bits, I * Bits->getBitWidth());
          continue;
        }
        return std::nullopt;
      }
      return SplatBits;
    }
  }
 
  return std::nullopt;
}
 
// Attempt to rebuild a normalized splat vector constant of the requested splat
// width, built up of potentially smaller scalar values.
// NOTE: We don't always bother converting to scalars if the vector length is 1.
static Constant *rebuildSplatableConstant(const Constant *C,
                                          unsigned SplatBitWidth) {
  std::optional<APInt> Splat = getSplatableConstant(C, SplatBitWidth);
  if (!Splat)
    return nullptr;
 
  // Determine scalar size to use for the constant splat vector, clamping as we
  // might have found a splat smaller than the original constant data.
  const Type *OriginalType = C->getType();
  Type *SclTy = OriginalType->getScalarType();
  unsigned NumSclBits = SclTy->getPrimitiveSizeInBits();
  NumSclBits = std::min<unsigned>(NumSclBits, SplatBitWidth);
 
  if (NumSclBits == 8) {
    SmallVector<uint8_t> RawBits;
    for (unsigned I = 0; I != SplatBitWidth; I += 8)
      RawBits.push_back(Splat->extractBits(8, I).getZExtValue());
    return ConstantDataVector::get(OriginalType->getContext(), RawBits);
  }
 
  if (NumSclBits == 16) {
    SmallVector<uint16_t> RawBits;
    for (unsigned I = 0; I != SplatBitWidth; I += 16)
      RawBits.push_back(Splat->extractBits(16, I).getZExtValue());
    if (SclTy->is16bitFPTy())
      return ConstantDataVector::getFP(SclTy, RawBits);
    return ConstantDataVector::get(OriginalType->getContext(), RawBits);
  }
 
  if (NumSclBits == 32) {
    SmallVector<uint32_t> RawBits;
    for (unsigned I = 0; I != SplatBitWidth; I += 32)
      RawBits.push_back(Splat->extractBits(32, I).getZExtValue());
    if (SclTy->isFloatTy())
      return ConstantDataVector::getFP(SclTy, RawBits);
    return ConstantDataVector::get(OriginalType->getContext(), RawBits);
  }
 
  // Fallback to i64 / double.
  SmallVector<uint64_t> RawBits;
  for (unsigned I = 0; I != SplatBitWidth; I += 64)
    RawBits.push_back(Splat->extractBits(64, I).getZExtValue());
  if (SclTy->isDoubleTy())
    return ConstantDataVector::getFP(SclTy, RawBits);
  return ConstantDataVector::get(OriginalType->getContext(), RawBits);
}
 
bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
                                                     MachineBasicBlock &MBB,
                                                     MachineInstr &MI) {
  unsigned Opc = MI.getOpcode();
  MachineConstantPool *CP = MI.getParent()->getParent()->getConstantPool();
  bool HasDQI = ST->hasDQI();
  bool HasBWI = ST->hasBWI();
 
  auto ConvertToBroadcast = [&](unsigned OpBcst256, unsigned OpBcst128,
                                unsigned OpBcst64, unsigned OpBcst32,
                                unsigned OpBcst16, unsigned OpBcst8,
                                unsigned OperandNo) {
    assert(MI.getNumOperands() >= (OperandNo + X86::AddrNumOperands) &&
           "Unexpected number of operands!");
 
    MachineOperand &CstOp = MI.getOperand(OperandNo + X86::AddrDisp);
    if (auto *C = getConstantFromPool(MI, CstOp)) {
      // Attempt to detect a suitable splat from increasing splat widths.
      std::pair<unsigned, unsigned> Broadcasts[] = {
          {8, OpBcst8},   {16, OpBcst16},   {32, OpBcst32},
          {64, OpBcst64}, {128, OpBcst128}, {256, OpBcst256},
      };
      for (auto [BitWidth, OpBcst] : Broadcasts) {
        if (OpBcst) {
          // Construct a suitable splat constant and adjust the MI to
          // use the new constant pool entry.
          if (Constant *NewCst = rebuildSplatableConstant(C, BitWidth)) {
            unsigned NewCPI =
                CP->getConstantPoolIndex(NewCst, Align(BitWidth / 8));
            MI.setDesc(TII->get(OpBcst));
            CstOp.setIndex(NewCPI);
            return true;
          }
        }
      }
    }
    return false;
  };
 
  // Attempt to convert full width vector loads into broadcast loads.
  switch (Opc) {
  /* FP Loads */
  case X86::MOVAPDrm:
  case X86::MOVAPSrm:
  case X86::MOVUPDrm:
  case X86::MOVUPSrm:
    // TODO: SSE3 MOVDDUP Handling
    return false;
  case X86::VMOVAPDrm:
  case X86::VMOVAPSrm:
  case X86::VMOVUPDrm:
  case X86::VMOVUPSrm:
    return ConvertToBroadcast(0, 0, X86::VMOVDDUPrm, X86::VBROADCASTSSrm, 0, 0,
                              1);
  case X86::VMOVAPDYrm:
  case X86::VMOVAPSYrm:
  case X86::VMOVUPDYrm:
  case X86::VMOVUPSYrm:
    return ConvertToBroadcast(0, X86::VBROADCASTF128, X86::VBROADCASTSDYrm,
                              X86::VBROADCASTSSYrm, 0, 0, 1);
  case X86::VMOVAPDZ128rm:
  case X86::VMOVAPSZ128rm:
  case X86::VMOVUPDZ128rm:
  case X86::VMOVUPSZ128rm:
    return ConvertToBroadcast(0, 0, X86::VMOVDDUPZ128rm,
                              X86::VBROADCASTSSZ128rm, 0, 0, 1);
  case X86::VMOVAPDZ256rm:
  case X86::VMOVAPSZ256rm:
  case X86::VMOVUPDZ256rm:
  case X86::VMOVUPSZ256rm:
    return ConvertToBroadcast(
        0, HasDQI ? X86::VBROADCASTF64X2Z128rm : X86::VBROADCASTF32X4Z256rm,
        X86::VBROADCASTSDZ256rm, X86::VBROADCASTSSZ256rm, 0, 0, 1);
  case X86::VMOVAPDZrm:
  case X86::VMOVAPSZrm:
  case X86::VMOVUPDZrm:
  case X86::VMOVUPSZrm:
    return ConvertToBroadcast(
        HasDQI ? X86::VBROADCASTF32X8rm : X86::VBROADCASTF64X4rm,
        HasDQI ? X86::VBROADCASTF64X2rm : X86::VBROADCASTF32X4rm,
        X86::VBROADCASTSDZrm, X86::VBROADCASTSSZrm, 0, 0, 1);
    /* Integer Loads */
  case X86::VMOVDQArm:
  case X86::VMOVDQUrm:
    if (ST->hasAVX2())
      return ConvertToBroadcast(0, 0, X86::VPBROADCASTQrm, X86::VPBROADCASTDrm,
                                X86::VPBROADCASTWrm, X86::VPBROADCASTBrm, 1);
    return ConvertToBroadcast(0, 0, X86::VMOVDDUPrm, X86::VBROADCASTSSrm, 0, 0,
                              1);
  case X86::VMOVDQAYrm:
  case X86::VMOVDQUYrm:
    if (ST->hasAVX2())
      return ConvertToBroadcast(0, X86::VBROADCASTI128, X86::VPBROADCASTQYrm,
                                X86::VPBROADCASTDYrm, X86::VPBROADCASTWYrm,
                                X86::VPBROADCASTBYrm, 1);
    return ConvertToBroadcast(0, X86::VBROADCASTF128, X86::VBROADCASTSDYrm,
                              X86::VBROADCASTSSYrm, 0, 0, 1);
  case X86::VMOVDQA32Z128rm:
  case X86::VMOVDQA64Z128rm:
  case X86::VMOVDQU32Z128rm:
  case X86::VMOVDQU64Z128rm:
    return ConvertToBroadcast(0, 0, X86::VPBROADCASTQZ128rm,
                              X86::VPBROADCASTDZ128rm,
                              HasBWI ? X86::VPBROADCASTWZ128rm : 0,
                              HasBWI ? X86::VPBROADCASTBZ128rm : 0, 1);
  case X86::VMOVDQA32Z256rm:
  case X86::VMOVDQA64Z256rm:
  case X86::VMOVDQU32Z256rm:
  case X86::VMOVDQU64Z256rm:
    return ConvertToBroadcast(
        0, HasDQI ? X86::VBROADCASTI64X2Z128rm : X86::VBROADCASTI32X4Z256rm,
        X86::VPBROADCASTQZ256rm, X86::VPBROADCASTDZ256rm,
        HasBWI ? X86::VPBROADCASTWZ256rm : 0,
        HasBWI ? X86::VPBROADCASTBZ256rm : 0, 1);
  case X86::VMOVDQA32Zrm:
  case X86::VMOVDQA64Zrm:
  case X86::VMOVDQU32Zrm:
  case X86::VMOVDQU64Zrm:
    return ConvertToBroadcast(
        HasDQI ? X86::VBROADCASTI32X8rm : X86::VBROADCASTI64X4rm,
        HasDQI ? X86::VBROADCASTI64X2rm : X86::VBROADCASTI32X4rm,
        X86::VPBROADCASTQZrm, X86::VPBROADCASTDZrm,
        HasBWI ? X86::VPBROADCASTWZrm : 0, HasBWI ? X86::VPBROADCASTBZrm : 0,
        1);
  }
 
  // Attempt to find a AVX512 mapping from a full width memory-fold instruction
  // to a broadcast-fold instruction variant.
  if ((MI.getDesc().TSFlags & X86II::EncodingMask) == X86II::EVEX) {
    unsigned OpBcst32 = 0, OpBcst64 = 0;
    unsigned OpNoBcst32 = 0, OpNoBcst64 = 0;
    if (const X86MemoryFoldTableEntry *Mem2Bcst =
            llvm::lookupBroadcastFoldTable(Opc, 32)) {
      OpBcst32 = Mem2Bcst->DstOp;
      OpNoBcst32 = Mem2Bcst->Flags & TB_INDEX_MASK;
    }
    if (const X86MemoryFoldTableEntry *Mem2Bcst =
            llvm::lookupBroadcastFoldTable(Opc, 64)) {
      OpBcst64 = Mem2Bcst->DstOp;
      OpNoBcst64 = Mem2Bcst->Flags & TB_INDEX_MASK;
    }
    assert(((OpBcst32 == 0) || (OpBcst64 == 0) || (OpNoBcst32 == OpNoBcst64)) &&
           "OperandNo mismatch");
 
    if (OpBcst32 || OpBcst64) {
      unsigned OpNo = OpBcst32 == 0 ? OpNoBcst64 : OpNoBcst32;
      return ConvertToBroadcast(0, 0, OpBcst64, OpBcst32, 0, 0, OpNo);
    }
  }
 
  return false;
}
 
bool X86FixupVectorConstantsPass::runOnMachineFunction(MachineFunction &MF) {
  LLVM_DEBUG(dbgs() << "Start X86FixupVectorConstants\n";);
  bool Changed = false;
  ST = &MF.getSubtarget<X86Subtarget>();
  TII = ST->getInstrInfo();
  SM = &ST->getSchedModel();
 
  for (MachineBasicBlock &MBB : MF) {
    for (MachineInstr &MI : MBB) {
      if (processInstruction(MF, MBB, MI)) {
        ++NumInstChanges;
        Changed = true;
      }
    }
  }
  LLVM_DEBUG(dbgs() << "End X86FixupVectorConstants\n";);
  return Changed;
}