|
LLVM 24.0.0git
|
This file implements the WebAssemblyTargetLowering class. More...
#include "WebAssemblyISelLowering.h"#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"#include "Utils/WebAssemblyTypeUtilities.h"#include "WebAssemblyMachineFunctionInfo.h"#include "WebAssemblySubtarget.h"#include "WebAssemblyTargetMachine.h"#include "WebAssemblyUtilities.h"#include "llvm/CodeGen/CallingConvLower.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineJumpTableInfo.h"#include "llvm/CodeGen/MachineModuleInfo.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/SDPatternMatch.h"#include "llvm/CodeGen/SelectionDAG.h"#include "llvm/CodeGen/SelectionDAGNodes.h"#include "llvm/IR/DiagnosticInfo.h"#include "llvm/IR/DiagnosticPrinter.h"#include "llvm/IR/Function.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/IntrinsicsWebAssembly.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/KnownBits.h"#include "llvm/Support/MathExtras.h"#include "llvm/Target/TargetOptions.h"Go to the source code of this file.
Classes | |
| struct | MaskReduceInfo |
Macros | |
| #define | DEBUG_TYPE "wasm-lower" |
This file implements the WebAssemblyTargetLowering class.
Definition in file WebAssemblyISelLowering.cpp.
| #define DEBUG_TYPE "wasm-lower" |
Definition at line 41 of file WebAssemblyISelLowering.cpp.
|
static |
Definition at line 1267 of file WebAssemblyISelLowering.cpp.
References llvm::CallingConv::C, llvm::CallingConv::Cold, llvm::CallingConv::CXX_FAST_TLS, llvm::CallingConv::Fast, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::WASM_EmscriptenInvoke.
|
static |
Definition at line 3741 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::AND, C(), CC, llvm::dyn_cast(), N, llvm::ISD::OR, llvm::ISD::SETEQ, and llvm::ISD::SETNE.
Referenced by performSETCCCombine().
|
static |
Definition at line 3669 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::XOR.
Referenced by performSETCCCombine().
|
static |
Try to convert a i128 comparison to a v16i8 comparison before type legalization splits it up into chunks.
Definition at line 3770 of file WebAssemblyISelLowering.cpp.
References CC, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::Function::hasFnAttribute(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), llvm::EVT::isByteSized(), llvm::EVT::isScalarInteger(), llvm::ISD::LOAD, N, llvm::peekThroughBitcasts(), SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETNE, Subtarget, X, and Y.
|
static |
Definition at line 3689 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::ElementCount::getFixed(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFixedLengthVector(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::ISD::XOR.
Referenced by performSETCCCombine().
|
static |
Definition at line 3093 of file WebAssemblyISelLowering.cpp.
References Concat, llvm::ISD::CONCAT_VECTORS, DL, DoubleVectorWidth(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::EVT::getVectorVT().
Referenced by DoubleVectorWidth(), ExtendLowAndHigh(), performConvertFPCombine(), and TryWideExtMulCombine().
|
static |
Definition at line 3108 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, DoubleVectorWidth(), and llvm::SelectionDAG::getNode().
Referenced by performVectorExtendCombine().
|
static |
Definition at line 3340 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), and llvm::SDNode::ops().
Referenced by combineBROADCAST_LOAD(), combineConcatVectorOps(), combineConstantPoolLoads(), combineEXTRACT_SUBVECTOR(), combineInt64DivRemViaFPReciprocal(), combineIntDivRemViaExactFPDiv(), combineLoad(), combineOr(), combineSCALAR_TO_VECTOR(), combineSelect(), combineTargetShuffle(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), extract128BitVector(), extract256BitVector(), getAVX512Node(), getAVX512TruncNode(), getEXTEND_VECTOR_INREG(), LowerBUILD_VECTORAsVariablePermute(), lowerBuildVectorAsBroadcast(), LowerEXTEND_VECTOR_INREG(), LowerPARTIAL_REDUCE_FMLA(), LowerPARTIAL_REDUCE_MLA(), lowerShuffleAsBroadcast(), lowerShuffleWithPERMV(), performBitcastCombine(), performConvertFPCombine(), performMulCombine(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), SplitOpsAndApply(), splitVector(), truncateVectorWithNARROW(), and truncateVectorWithPACK().
|
static |
Definition at line 1260 of file WebAssemblyISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and Msg.
|
static |
Definition at line 2373 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::cast(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), Opc, llvm::peekThroughBitcasts(), SDValue(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by TryWideExtMulCombine().
|
static |
Definition at line 2991 of file WebAssemblyISelLowering.cpp.
References llvm::SelectionDAG::isKnownNeverLogicalZero(), and llvm::SelectionDAG::isKnownNeverNaN().
Definition at line 1860 of file WebAssemblyISelLowering.cpp.
References llvm::dyn_cast(), llvm::GlobalAddressSDNode::getAddressSpace(), and llvm::WebAssembly::isWasmVarAddressSpace().
|
static |
Definition at line 1867 of file WebAssemblyISelLowering.cpp.
References llvm::dyn_cast(), llvm::FrameIndexSDNode::getIndex(), llvm::WebAssemblyFrameLowering::getLocalForStackObject(), and llvm::SelectionDAG::getMachineFunction().
|
static |
Definition at line 787 of file WebAssemblyISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstr::addOperand(), llvm::MachineInstrBuilder::addReg(), llvm::MachineInstrBuilder::addSym(), assert(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::MachineInstr::defs(), DL, llvm::MachineInstr::eraseFromParent(), llvm::MachineInstr::explicit_uses(), llvm::MachineFunction::getContext(), llvm::MachineInstrBuilder::getInstr(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::WebAssembly::getOrCreateFuncrefCallTableSymbol(), llvm::WebAssembly::getOrCreateFunctionTableSymbol(), llvm::MachineBasicBlock::getParent(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getPrevNode(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineBasicBlock::insert(), llvm::MachineBasicBlock::insertAfter(), llvm::MachineOperand::isFI(), llvm::MachineOperand::isReg(), MRI, Reg, llvm::MachineInstr::removeOperand(), Subtarget, llvm::Table, and TII.
|
static |
Definition at line 2455 of file WebAssemblyISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP16_TO_FP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::isa(), SDValue(), llvm::ISD::SINT_TO_FP, and llvm::ISD::UINT_TO_FP.
|
static |
Definition at line 516 of file WebAssemblyISelLowering.cpp.
References llvm::MachineInstrBuilder::addFPImm(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::cast(), DL, llvm::MachineBasicBlock::end(), F, llvm::MachineBasicBlock::getBasicBlock(), llvm::Function::getContext(), llvm::Type::getDoubleTy(), llvm::Type::getFloatTy(), llvm::MachineFunction::getFunction(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), INT64_MIN, MI, MRI, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
|
static |
Definition at line 611 of file WebAssemblyISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), F, llvm::MachineBasicBlock::getBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), MI, MRI, llvm::MachineOperand::setIsKill(), llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
|
static |
Definition at line 703 of file WebAssemblyISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), F, llvm::MachineBasicBlock::getBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), MI, MRI, llvm::MachineOperand::setIsKill(), llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
|
static |
Definition at line 3611 of file WebAssemblyISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, llvm::SDPatternMatch::m_c_SetCC(), llvm::SDPatternMatch::m_c_SpecificSetCC(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), N, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::XOR.
|
static |
Definition at line 3488 of file WebAssemblyISelLowering.cpp.
References assert(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::ElementCount::getFixed(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVectorOf(), llvm::EVT::isScalarInteger(), llvm::PatternMatch::m_BitCast(), llvm::SDPatternMatch::m_c_SetCC(), llvm::PatternMatch::m_Value(), N, llvm::ISD::OR, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SHL, and llvm::size().
|
static |
Definition at line 3592 of file WebAssemblyISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, llvm::SDPatternMatch::m_c_SpecificSetCC(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), N, llvm::SDPatternMatch::sd_match(), SDValue(), and llvm::ISD::SETLT.
| SDValue performConvertFPCombine | ( | SDNode * | N, |
| SelectionDAG & | DAG ) |
Definition at line 4004 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::AND, DL, DoubleVectorWidth(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::Masked, N, SDValue(), and truncateVectorWithNARROW().
|
static |
Definition at line 4117 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), and TLI.
|
static |
Definition at line 3946 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::isVector(), LHS, llvm::ISD::MUL, N, RHS, SDValue(), and TryWideExtMulCombine().
Definition at line 4139 of file WebAssemblyISelLowering.cpp.
|
static |
Definition at line 3814 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::BITCAST, classifyMaskReduction(), combineSmallMaskReduction(), combineVectorSizedSetCCEquality(), combineWideMaskReduction(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::EVT::getVectorNumElements(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVectorOf(), llvm::EVT::isScalarInteger(), LHS, N, SDValue(), Subtarget, and V.
|
static |
Definition at line 4059 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::BitWidth, llvm::ISD::BUILD_VECTOR, C(), llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), I, LHS, llvm::ISD::MUL, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, SDValue(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::APInt::uge(), and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 3464 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::AND, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::isSimple(), llvm::EVT::isVector(), N, SDValue(), and truncateVectorWithNARROW().
|
static |
Definition at line 3022 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), N, and SDValue().
|
static |
Definition at line 3142 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), ExtendLowAndHigh(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), N, SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
|
static |
Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get split up into scalar instructions during legalization, and the vector extending instructions are selected in performVectorExtendCombine below.
Definition at line 3050 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, Subtarget, llvm::ISD::UINT_TO_FP, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 3075 of file WebAssemblyISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::EVT::isVector(), N, SDValue(), llvm::SelectionDAG::SignBitIsZero(), and llvm::ISD::SINT_TO_FP.
|
static |
Definition at line 3199 of file WebAssemblyISelLowering.cpp.
References Concat, llvm::ISD::CONCAT_VECTORS, Conversion, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::ISD::FP_ROUND, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getNode(), llvm_unreachable, N, SDValue(), and llvm::Splat.
|
static |
Definition at line 3369 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is32BitVector(), llvm::EVT::is64BitVector(), llvm::isPowerOf2_32(), Lo, SDValue(), truncateVectorWithNARROW(), and llvm::EVT::widenIntegerVectorElementType().
Referenced by performConvertFPCombine(), performTruncateCombine(), and truncateVectorWithNARROW().
|
static |
Definition at line 3851 of file WebAssemblyISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, DoubleVectorWidth(), GetExtendHigh(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::Hi, LHS, Lo, llvm::ISD::MUL, N, RHS, SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performMulCombine().
|
static |
Definition at line 2882 of file WebAssemblyISelLowering.cpp.
References llvm::ISD::AND, llvm::EVT::bitsGE(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getValueType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::SelectionDAG::UnrollVectorOp().