| File: | build/source/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp |
| Warning: | line 300, column 9 Assigned value is garbage or undefined |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===-- BPFISelDAGToDAG.cpp - A dag to dag inst selector for BPF ----------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // This file defines a DAG pattern matching instruction selector for BPF, | ||||||||
| 10 | // converting from a legalized dag to a BPF dag. | ||||||||
| 11 | // | ||||||||
| 12 | //===----------------------------------------------------------------------===// | ||||||||
| 13 | |||||||||
| 14 | #include "BPF.h" | ||||||||
| 15 | #include "BPFRegisterInfo.h" | ||||||||
| 16 | #include "BPFSubtarget.h" | ||||||||
| 17 | #include "BPFTargetMachine.h" | ||||||||
| 18 | #include "llvm/CodeGen/FunctionLoweringInfo.h" | ||||||||
| 19 | #include "llvm/CodeGen/MachineConstantPool.h" | ||||||||
| 20 | #include "llvm/CodeGen/MachineFrameInfo.h" | ||||||||
| 21 | #include "llvm/CodeGen/MachineFunction.h" | ||||||||
| 22 | #include "llvm/CodeGen/MachineInstrBuilder.h" | ||||||||
| 23 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||||||
| 24 | #include "llvm/CodeGen/SelectionDAGISel.h" | ||||||||
| 25 | #include "llvm/IR/Constants.h" | ||||||||
| 26 | #include "llvm/IR/IntrinsicInst.h" | ||||||||
| 27 | #include "llvm/IR/IntrinsicsBPF.h" | ||||||||
| 28 | #include "llvm/Support/Debug.h" | ||||||||
| 29 | #include "llvm/Support/Endian.h" | ||||||||
| 30 | #include "llvm/Support/ErrorHandling.h" | ||||||||
| 31 | #include "llvm/Support/raw_ostream.h" | ||||||||
| 32 | #include "llvm/Target/TargetMachine.h" | ||||||||
| 33 | |||||||||
| 34 | using namespace llvm; | ||||||||
| 35 | |||||||||
| 36 | #define DEBUG_TYPE"bpf-isel" "bpf-isel" | ||||||||
| 37 | #define PASS_NAME"BPF DAG->DAG Pattern Instruction Selection" "BPF DAG->DAG Pattern Instruction Selection" | ||||||||
| 38 | |||||||||
| 39 | // Instruction Selector Implementation | ||||||||
| 40 | namespace { | ||||||||
| 41 | |||||||||
| 42 | class BPFDAGToDAGISel : public SelectionDAGISel { | ||||||||
| 43 | |||||||||
| 44 | /// Subtarget - Keep a pointer to the BPFSubtarget around so that we can | ||||||||
| 45 | /// make the right decision when generating code for different subtargets. | ||||||||
| 46 | const BPFSubtarget *Subtarget; | ||||||||
| 47 | |||||||||
| 48 | public: | ||||||||
| 49 | static char ID; | ||||||||
| 50 | |||||||||
| 51 | BPFDAGToDAGISel() = delete; | ||||||||
| 52 | |||||||||
| 53 | explicit BPFDAGToDAGISel(BPFTargetMachine &TM) | ||||||||
| 54 | : SelectionDAGISel(ID, TM), Subtarget(nullptr) {} | ||||||||
| 55 | |||||||||
| 56 | bool runOnMachineFunction(MachineFunction &MF) override { | ||||||||
| 57 | // Reset the subtarget each time through. | ||||||||
| 58 | Subtarget = &MF.getSubtarget<BPFSubtarget>(); | ||||||||
| 59 | return SelectionDAGISel::runOnMachineFunction(MF); | ||||||||
| 60 | } | ||||||||
| 61 | |||||||||
| 62 | void PreprocessISelDAG() override; | ||||||||
| 63 | |||||||||
| 64 | bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintCode, | ||||||||
| 65 | std::vector<SDValue> &OutOps) override; | ||||||||
| 66 | |||||||||
| 67 | |||||||||
| 68 | private: | ||||||||
| 69 | // Include the pieces autogenerated from the target description. | ||||||||
| 70 | #include "BPFGenDAGISel.inc" | ||||||||
| 71 | |||||||||
| 72 | void Select(SDNode *N) override; | ||||||||
| 73 | |||||||||
| 74 | // Complex Pattern for address selection. | ||||||||
| 75 | bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset); | ||||||||
| 76 | bool SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset); | ||||||||
| 77 | |||||||||
| 78 | // Node preprocessing cases | ||||||||
| 79 | void PreprocessLoad(SDNode *Node, SelectionDAG::allnodes_iterator &I); | ||||||||
| 80 | void PreprocessCopyToReg(SDNode *Node); | ||||||||
| 81 | void PreprocessTrunc(SDNode *Node, SelectionDAG::allnodes_iterator &I); | ||||||||
| 82 | |||||||||
| 83 | // Find constants from a constant structure | ||||||||
| 84 | typedef std::vector<unsigned char> val_vec_type; | ||||||||
| 85 | bool fillGenericConstant(const DataLayout &DL, const Constant *CV, | ||||||||
| 86 | val_vec_type &Vals, uint64_t Offset); | ||||||||
| 87 | bool fillConstantDataArray(const DataLayout &DL, const ConstantDataArray *CDA, | ||||||||
| 88 | val_vec_type &Vals, int Offset); | ||||||||
| 89 | bool fillConstantArray(const DataLayout &DL, const ConstantArray *CA, | ||||||||
| 90 | val_vec_type &Vals, int Offset); | ||||||||
| 91 | bool fillConstantStruct(const DataLayout &DL, const ConstantStruct *CS, | ||||||||
| 92 | val_vec_type &Vals, int Offset); | ||||||||
| 93 | bool getConstantFieldValue(const GlobalAddressSDNode *Node, uint64_t Offset, | ||||||||
| 94 | uint64_t Size, unsigned char *ByteSeq); | ||||||||
| 95 | // Mapping from ConstantStruct global value to corresponding byte-list values | ||||||||
| 96 | std::map<const void *, val_vec_type> cs_vals_; | ||||||||
| 97 | }; | ||||||||
| 98 | } // namespace | ||||||||
| 99 | |||||||||
| 100 | char BPFDAGToDAGISel::ID = 0; | ||||||||
| 101 | |||||||||
| 102 | INITIALIZE_PASS(BPFDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)static void *initializeBPFDAGToDAGISelPassOnce(PassRegistry & Registry) { PassInfo *PI = new PassInfo( "BPF DAG->DAG Pattern Instruction Selection" , "bpf-isel", &BPFDAGToDAGISel::ID, PassInfo::NormalCtor_t (callDefaultCtor<BPFDAGToDAGISel>), false, false); Registry .registerPass(*PI, true); return PI; } static llvm::once_flag InitializeBPFDAGToDAGISelPassFlag; void llvm::initializeBPFDAGToDAGISelPass (PassRegistry &Registry) { llvm::call_once(InitializeBPFDAGToDAGISelPassFlag , initializeBPFDAGToDAGISelPassOnce, std::ref(Registry)); } | ||||||||
| 103 | |||||||||
| 104 | // ComplexPattern used on BPF Load/Store instructions | ||||||||
| 105 | bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) { | ||||||||
| 106 | // if Address is FI, get the TargetFrameIndex. | ||||||||
| 107 | SDLoc DL(Addr); | ||||||||
| 108 | if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { | ||||||||
| 109 | Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); | ||||||||
| 110 | Offset = CurDAG->getTargetConstant(0, DL, MVT::i64); | ||||||||
| 111 | return true; | ||||||||
| 112 | } | ||||||||
| 113 | |||||||||
| 114 | if (Addr.getOpcode() == ISD::TargetExternalSymbol || | ||||||||
| 115 | Addr.getOpcode() == ISD::TargetGlobalAddress) | ||||||||
| 116 | return false; | ||||||||
| 117 | |||||||||
| 118 | // Addresses of the form Addr+const or Addr|const | ||||||||
| 119 | if (CurDAG->isBaseWithConstantOffset(Addr)) { | ||||||||
| 120 | auto *CN = cast<ConstantSDNode>(Addr.getOperand(1)); | ||||||||
| 121 | if (isInt<16>(CN->getSExtValue())) { | ||||||||
| 122 | // If the first operand is a FI, get the TargetFI Node | ||||||||
| 123 | if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) | ||||||||
| 124 | Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); | ||||||||
| 125 | else | ||||||||
| 126 | Base = Addr.getOperand(0); | ||||||||
| 127 | |||||||||
| 128 | Offset = CurDAG->getTargetConstant(CN->getSExtValue(), DL, MVT::i64); | ||||||||
| 129 | return true; | ||||||||
| 130 | } | ||||||||
| 131 | } | ||||||||
| 132 | |||||||||
| 133 | Base = Addr; | ||||||||
| 134 | Offset = CurDAG->getTargetConstant(0, DL, MVT::i64); | ||||||||
| 135 | return true; | ||||||||
| 136 | } | ||||||||
| 137 | |||||||||
| 138 | // ComplexPattern used on BPF FI instruction | ||||||||
| 139 | bool BPFDAGToDAGISel::SelectFIAddr(SDValue Addr, SDValue &Base, | ||||||||
| 140 | SDValue &Offset) { | ||||||||
| 141 | SDLoc DL(Addr); | ||||||||
| 142 | |||||||||
| 143 | if (!CurDAG->isBaseWithConstantOffset(Addr)) | ||||||||
| 144 | return false; | ||||||||
| 145 | |||||||||
| 146 | // Addresses of the form Addr+const or Addr|const | ||||||||
| 147 | auto *CN = cast<ConstantSDNode>(Addr.getOperand(1)); | ||||||||
| 148 | if (isInt<16>(CN->getSExtValue())) { | ||||||||
| 149 | // If the first operand is a FI, get the TargetFI Node | ||||||||
| 150 | if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) | ||||||||
| 151 | Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); | ||||||||
| 152 | else | ||||||||
| 153 | return false; | ||||||||
| 154 | |||||||||
| 155 | Offset = CurDAG->getTargetConstant(CN->getSExtValue(), DL, MVT::i64); | ||||||||
| 156 | return true; | ||||||||
| 157 | } | ||||||||
| 158 | |||||||||
| 159 | return false; | ||||||||
| 160 | } | ||||||||
| 161 | |||||||||
| 162 | bool BPFDAGToDAGISel::SelectInlineAsmMemoryOperand( | ||||||||
| 163 | const SDValue &Op, unsigned ConstraintCode, std::vector<SDValue> &OutOps) { | ||||||||
| 164 | SDValue Op0, Op1; | ||||||||
| 165 | switch (ConstraintCode) { | ||||||||
| 166 | default: | ||||||||
| 167 | return true; | ||||||||
| 168 | case InlineAsm::Constraint_m: // memory | ||||||||
| 169 | if (!SelectAddr(Op, Op0, Op1)) | ||||||||
| 170 | return true; | ||||||||
| 171 | break; | ||||||||
| 172 | } | ||||||||
| 173 | |||||||||
| 174 | SDLoc DL(Op); | ||||||||
| 175 | SDValue AluOp = CurDAG->getTargetConstant(ISD::ADD, DL, MVT::i32);; | ||||||||
| 176 | OutOps.push_back(Op0); | ||||||||
| 177 | OutOps.push_back(Op1); | ||||||||
| 178 | OutOps.push_back(AluOp); | ||||||||
| 179 | return false; | ||||||||
| 180 | } | ||||||||
| 181 | |||||||||
| 182 | void BPFDAGToDAGISel::Select(SDNode *Node) { | ||||||||
| 183 | unsigned Opcode = Node->getOpcode(); | ||||||||
| 184 | |||||||||
| 185 | // If we have a custom node, we already have selected! | ||||||||
| 186 | if (Node->isMachineOpcode()) { | ||||||||
| 187 | LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'; } } while (false); | ||||||||
| 188 | return; | ||||||||
| 189 | } | ||||||||
| 190 | |||||||||
| 191 | // tablegen selection should be handled here. | ||||||||
| 192 | switch (Opcode) { | ||||||||
| 193 | default: | ||||||||
| 194 | break; | ||||||||
| 195 | case ISD::SDIV: { | ||||||||
| 196 | DebugLoc Empty; | ||||||||
| 197 | const DebugLoc &DL = Node->getDebugLoc(); | ||||||||
| 198 | if (DL != Empty) | ||||||||
| 199 | errs() << "Error at line " << DL.getLine() << ": "; | ||||||||
| 200 | else | ||||||||
| 201 | errs() << "Error: "; | ||||||||
| 202 | errs() << "Unsupport signed division for DAG: "; | ||||||||
| 203 | Node->print(errs(), CurDAG); | ||||||||
| 204 | errs() << "Please convert to unsigned div/mod.\n"; | ||||||||
| 205 | break; | ||||||||
| 206 | } | ||||||||
| 207 | case ISD::INTRINSIC_W_CHAIN: { | ||||||||
| 208 | unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue(); | ||||||||
| 209 | switch (IntNo) { | ||||||||
| 210 | case Intrinsic::bpf_load_byte: | ||||||||
| 211 | case Intrinsic::bpf_load_half: | ||||||||
| 212 | case Intrinsic::bpf_load_word: { | ||||||||
| 213 | SDLoc DL(Node); | ||||||||
| 214 | SDValue Chain = Node->getOperand(0); | ||||||||
| 215 | SDValue N1 = Node->getOperand(1); | ||||||||
| 216 | SDValue Skb = Node->getOperand(2); | ||||||||
| 217 | SDValue N3 = Node->getOperand(3); | ||||||||
| 218 | |||||||||
| 219 | SDValue R6Reg = CurDAG->getRegister(BPF::R6, MVT::i64); | ||||||||
| 220 | Chain = CurDAG->getCopyToReg(Chain, DL, R6Reg, Skb, SDValue()); | ||||||||
| 221 | Node = CurDAG->UpdateNodeOperands(Node, Chain, N1, R6Reg, N3); | ||||||||
| 222 | break; | ||||||||
| 223 | } | ||||||||
| 224 | } | ||||||||
| 225 | break; | ||||||||
| 226 | } | ||||||||
| 227 | |||||||||
| 228 | case ISD::FrameIndex: { | ||||||||
| 229 | int FI = cast<FrameIndexSDNode>(Node)->getIndex(); | ||||||||
| 230 | EVT VT = Node->getValueType(0); | ||||||||
| 231 | SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT); | ||||||||
| 232 | unsigned Opc = BPF::MOV_rr; | ||||||||
| 233 | if (Node->hasOneUse()) { | ||||||||
| 234 | CurDAG->SelectNodeTo(Node, Opc, VT, TFI); | ||||||||
| 235 | return; | ||||||||
| 236 | } | ||||||||
| 237 | ReplaceNode(Node, CurDAG->getMachineNode(Opc, SDLoc(Node), VT, TFI)); | ||||||||
| 238 | return; | ||||||||
| 239 | } | ||||||||
| 240 | } | ||||||||
| 241 | |||||||||
| 242 | // Select the default instruction | ||||||||
| 243 | SelectCode(Node); | ||||||||
| 244 | } | ||||||||
| 245 | |||||||||
| 246 | void BPFDAGToDAGISel::PreprocessLoad(SDNode *Node, | ||||||||
| 247 | SelectionDAG::allnodes_iterator &I) { | ||||||||
| 248 | union { | ||||||||
| 249 | uint8_t c[8]; | ||||||||
| 250 | uint16_t s; | ||||||||
| 251 | uint32_t i; | ||||||||
| 252 | uint64_t d; | ||||||||
| 253 | } new_val; // hold up the constant values replacing loads. | ||||||||
| 254 | bool to_replace = false; | ||||||||
| 255 | SDLoc DL(Node); | ||||||||
| 256 | const LoadSDNode *LD = cast<LoadSDNode>(Node); | ||||||||
| 257 | uint64_t size = LD->getMemOperand()->getSize(); | ||||||||
| 258 | |||||||||
| 259 | if (!size || size > 8 || (size & (size - 1)) || !LD->isSimple()) | ||||||||
| 260 | return; | ||||||||
| 261 | |||||||||
| 262 | SDNode *LDAddrNode = LD->getOperand(1).getNode(); | ||||||||
| 263 | // Match LDAddr against either global_addr or (global_addr + offset) | ||||||||
| 264 | unsigned opcode = LDAddrNode->getOpcode(); | ||||||||
| 265 | if (opcode == ISD::ADD) { | ||||||||
| 266 | SDValue OP1 = LDAddrNode->getOperand(0); | ||||||||
| 267 | SDValue OP2 = LDAddrNode->getOperand(1); | ||||||||
| 268 | |||||||||
| 269 | // We want to find the pattern global_addr + offset | ||||||||
| 270 | SDNode *OP1N = OP1.getNode(); | ||||||||
| 271 | if (OP1N->getOpcode() <= ISD::BUILTIN_OP_END || OP1N->getNumOperands() == 0) | ||||||||
| 272 | return; | ||||||||
| 273 | |||||||||
| 274 | LLVM_DEBUG(dbgs() << "Check candidate load: "; LD->dump(); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Check candidate load: "; LD-> dump(); dbgs() << '\n'; } } while (false); | ||||||||
| 275 | |||||||||
| 276 | const GlobalAddressSDNode *GADN = | ||||||||
| 277 | dyn_cast<GlobalAddressSDNode>(OP1N->getOperand(0).getNode()); | ||||||||
| 278 | const ConstantSDNode *CDN = dyn_cast<ConstantSDNode>(OP2.getNode()); | ||||||||
| 279 | if (GADN
| ||||||||
| 280 | to_replace = | ||||||||
| 281 | getConstantFieldValue(GADN, CDN->getZExtValue(), size, new_val.c); | ||||||||
| 282 | } else if (LDAddrNode->getOpcode() > ISD::BUILTIN_OP_END && | ||||||||
| 283 | LDAddrNode->getNumOperands() > 0) { | ||||||||
| 284 | LLVM_DEBUG(dbgs() << "Check candidate load: "; LD->dump(); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Check candidate load: "; LD-> dump(); dbgs() << '\n'; } } while (false); | ||||||||
| 285 | |||||||||
| 286 | SDValue OP1 = LDAddrNode->getOperand(0); | ||||||||
| 287 | if (const GlobalAddressSDNode *GADN = | ||||||||
| 288 | dyn_cast<GlobalAddressSDNode>(OP1.getNode())) | ||||||||
| 289 | to_replace = getConstantFieldValue(GADN, 0, size, new_val.c); | ||||||||
| 290 | } | ||||||||
| 291 | |||||||||
| 292 | if (!to_replace
| ||||||||
| 293 | return; | ||||||||
| 294 | |||||||||
| 295 | // replacing the old with a new value | ||||||||
| 296 | uint64_t val; | ||||||||
| 297 | if (size == 1) | ||||||||
| 298 | val = new_val.c[0]; | ||||||||
| 299 | else if (size == 2) | ||||||||
| 300 | val = new_val.s; | ||||||||
| |||||||||
| 301 | else if (size == 4) | ||||||||
| 302 | val = new_val.i; | ||||||||
| 303 | else { | ||||||||
| 304 | val = new_val.d; | ||||||||
| 305 | } | ||||||||
| 306 | |||||||||
| 307 | LLVM_DEBUG(dbgs() << "Replacing load of size " << size << " with constant "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Replacing load of size " << size << " with constant " << val << '\n'; } } while (false) | ||||||||
| 308 | << val << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Replacing load of size " << size << " with constant " << val << '\n'; } } while (false); | ||||||||
| 309 | SDValue NVal = CurDAG->getConstant(val, DL, LD->getValueType(0)); | ||||||||
| 310 | |||||||||
| 311 | // After replacement, the current node is dead, we need to | ||||||||
| 312 | // go backward one step to make iterator still work | ||||||||
| 313 | I--; | ||||||||
| 314 | SDValue From[] = {SDValue(Node, 0), SDValue(Node, 1)}; | ||||||||
| 315 | SDValue To[] = {NVal, NVal}; | ||||||||
| 316 | CurDAG->ReplaceAllUsesOfValuesWith(From, To, 2); | ||||||||
| 317 | I++; | ||||||||
| 318 | // It is safe to delete node now | ||||||||
| 319 | CurDAG->DeleteNode(Node); | ||||||||
| 320 | } | ||||||||
| 321 | |||||||||
| 322 | void BPFDAGToDAGISel::PreprocessISelDAG() { | ||||||||
| 323 | // Iterate through all nodes, interested in the following case: | ||||||||
| 324 | // | ||||||||
| 325 | // . loads from ConstantStruct or ConstantArray of constructs | ||||||||
| 326 | // which can be turns into constant itself, with this we can | ||||||||
| 327 | // avoid reading from read-only section at runtime. | ||||||||
| 328 | // | ||||||||
| 329 | // . Removing redundant AND for intrinsic narrow loads. | ||||||||
| 330 | for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), | ||||||||
| |||||||||
| 331 | E = CurDAG->allnodes_end(); | ||||||||
| 332 | I != E;) { | ||||||||
| 333 | SDNode *Node = &*I++; | ||||||||
| 334 | unsigned Opcode = Node->getOpcode(); | ||||||||
| 335 | if (Opcode == ISD::LOAD) | ||||||||
| 336 | PreprocessLoad(Node, I); | ||||||||
| 337 | else if (Opcode == ISD::AND) | ||||||||
| 338 | PreprocessTrunc(Node, I); | ||||||||
| 339 | } | ||||||||
| 340 | } | ||||||||
| 341 | |||||||||
| 342 | bool BPFDAGToDAGISel::getConstantFieldValue(const GlobalAddressSDNode *Node, | ||||||||
| 343 | uint64_t Offset, uint64_t Size, | ||||||||
| 344 | unsigned char *ByteSeq) { | ||||||||
| 345 | const GlobalVariable *V = dyn_cast<GlobalVariable>(Node->getGlobal()); | ||||||||
| 346 | |||||||||
| 347 | if (!V
| ||||||||
| 348 | return false; | ||||||||
| 349 | |||||||||
| 350 | const Constant *Init = V->getInitializer(); | ||||||||
| 351 | const DataLayout &DL = CurDAG->getDataLayout(); | ||||||||
| 352 | val_vec_type TmpVal; | ||||||||
| 353 | |||||||||
| 354 | auto it = cs_vals_.find(static_cast<const void *>(Init)); | ||||||||
| 355 | if (it != cs_vals_.end()) { | ||||||||
| 356 | TmpVal = it->second; | ||||||||
| 357 | } else { | ||||||||
| 358 | uint64_t total_size = 0; | ||||||||
| 359 | if (const ConstantStruct *CS = dyn_cast<ConstantStruct>(Init)) | ||||||||
| 360 | total_size = | ||||||||
| 361 | DL.getStructLayout(cast<StructType>(CS->getType()))->getSizeInBytes(); | ||||||||
| 362 | else if (const ConstantArray *CA = dyn_cast<ConstantArray>(Init)) | ||||||||
| 363 | total_size = DL.getTypeAllocSize(CA->getType()->getElementType()) * | ||||||||
| 364 | CA->getNumOperands(); | ||||||||
| 365 | else | ||||||||
| 366 | return false; | ||||||||
| 367 | |||||||||
| 368 | val_vec_type Vals(total_size, 0); | ||||||||
| 369 | if (fillGenericConstant(DL, Init, Vals, 0) == false) | ||||||||
| 370 | return false; | ||||||||
| 371 | cs_vals_[static_cast<const void *>(Init)] = Vals; | ||||||||
| 372 | TmpVal = std::move(Vals); | ||||||||
| 373 | } | ||||||||
| 374 | |||||||||
| 375 | // test whether host endianness matches target | ||||||||
| 376 | union { | ||||||||
| 377 | uint8_t c[2]; | ||||||||
| 378 | uint16_t s; | ||||||||
| 379 | } test_buf; | ||||||||
| 380 | uint16_t test_val = 0x2345; | ||||||||
| 381 | if (DL.isLittleEndian()) | ||||||||
| 382 | support::endian::write16le(test_buf.c, test_val); | ||||||||
| 383 | else | ||||||||
| 384 | support::endian::write16be(test_buf.c, test_val); | ||||||||
| 385 | |||||||||
| 386 | bool endian_match = test_buf.s == test_val; | ||||||||
| 387 | for (uint64_t i = Offset, j = 0; i < Offset + Size; i++, j++) | ||||||||
| 388 | ByteSeq[j] = endian_match ? TmpVal[i] : TmpVal[Offset + Size - 1 - j]; | ||||||||
| 389 | |||||||||
| 390 | return true; | ||||||||
| 391 | } | ||||||||
| 392 | |||||||||
| 393 | bool BPFDAGToDAGISel::fillGenericConstant(const DataLayout &DL, | ||||||||
| 394 | const Constant *CV, | ||||||||
| 395 | val_vec_type &Vals, uint64_t Offset) { | ||||||||
| 396 | uint64_t Size = DL.getTypeAllocSize(CV->getType()); | ||||||||
| 397 | |||||||||
| 398 | if (isa<ConstantAggregateZero>(CV) || isa<UndefValue>(CV)) | ||||||||
| 399 | return true; // already done | ||||||||
| 400 | |||||||||
| 401 | if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) { | ||||||||
| 402 | uint64_t val = CI->getZExtValue(); | ||||||||
| 403 | LLVM_DEBUG(dbgs() << "Byte array at offset " << Offset << " with value "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Byte array at offset " << Offset << " with value " << val << '\n'; } } while (false) | ||||||||
| 404 | << val << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Byte array at offset " << Offset << " with value " << val << '\n'; } } while (false); | ||||||||
| 405 | |||||||||
| 406 | if (Size > 8 || (Size & (Size - 1))) | ||||||||
| 407 | return false; | ||||||||
| 408 | |||||||||
| 409 | // Store based on target endian | ||||||||
| 410 | for (uint64_t i = 0; i < Size; ++i) { | ||||||||
| 411 | Vals[Offset + i] = DL.isLittleEndian() | ||||||||
| 412 | ? ((val >> (i * 8)) & 0xFF) | ||||||||
| 413 | : ((val >> ((Size - i - 1) * 8)) & 0xFF); | ||||||||
| 414 | } | ||||||||
| 415 | return true; | ||||||||
| 416 | } | ||||||||
| 417 | |||||||||
| 418 | if (const ConstantDataArray *CDA = dyn_cast<ConstantDataArray>(CV)) | ||||||||
| 419 | return fillConstantDataArray(DL, CDA, Vals, Offset); | ||||||||
| 420 | |||||||||
| 421 | if (const ConstantArray *CA = dyn_cast<ConstantArray>(CV)) | ||||||||
| 422 | return fillConstantArray(DL, CA, Vals, Offset); | ||||||||
| 423 | |||||||||
| 424 | if (const ConstantStruct *CVS = dyn_cast<ConstantStruct>(CV)) | ||||||||
| 425 | return fillConstantStruct(DL, CVS, Vals, Offset); | ||||||||
| 426 | |||||||||
| 427 | return false; | ||||||||
| 428 | } | ||||||||
| 429 | |||||||||
| 430 | bool BPFDAGToDAGISel::fillConstantDataArray(const DataLayout &DL, | ||||||||
| 431 | const ConstantDataArray *CDA, | ||||||||
| 432 | val_vec_type &Vals, int Offset) { | ||||||||
| 433 | for (unsigned i = 0, e = CDA->getNumElements(); i != e; ++i) { | ||||||||
| 434 | if (fillGenericConstant(DL, CDA->getElementAsConstant(i), Vals, Offset) == | ||||||||
| 435 | false) | ||||||||
| 436 | return false; | ||||||||
| 437 | Offset += DL.getTypeAllocSize(CDA->getElementAsConstant(i)->getType()); | ||||||||
| 438 | } | ||||||||
| 439 | |||||||||
| 440 | return true; | ||||||||
| 441 | } | ||||||||
| 442 | |||||||||
| 443 | bool BPFDAGToDAGISel::fillConstantArray(const DataLayout &DL, | ||||||||
| 444 | const ConstantArray *CA, | ||||||||
| 445 | val_vec_type &Vals, int Offset) { | ||||||||
| 446 | for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i) { | ||||||||
| 447 | if (fillGenericConstant(DL, CA->getOperand(i), Vals, Offset) == false) | ||||||||
| 448 | return false; | ||||||||
| 449 | Offset += DL.getTypeAllocSize(CA->getOperand(i)->getType()); | ||||||||
| 450 | } | ||||||||
| 451 | |||||||||
| 452 | return true; | ||||||||
| 453 | } | ||||||||
| 454 | |||||||||
| 455 | bool BPFDAGToDAGISel::fillConstantStruct(const DataLayout &DL, | ||||||||
| 456 | const ConstantStruct *CS, | ||||||||
| 457 | val_vec_type &Vals, int Offset) { | ||||||||
| 458 | const StructLayout *Layout = DL.getStructLayout(CS->getType()); | ||||||||
| 459 | for (unsigned i = 0, e = CS->getNumOperands(); i != e; ++i) { | ||||||||
| 460 | const Constant *Field = CS->getOperand(i); | ||||||||
| 461 | uint64_t SizeSoFar = Layout->getElementOffset(i); | ||||||||
| 462 | if (fillGenericConstant(DL, Field, Vals, Offset + SizeSoFar) == false) | ||||||||
| 463 | return false; | ||||||||
| 464 | } | ||||||||
| 465 | return true; | ||||||||
| 466 | } | ||||||||
| 467 | |||||||||
| 468 | void BPFDAGToDAGISel::PreprocessTrunc(SDNode *Node, | ||||||||
| 469 | SelectionDAG::allnodes_iterator &I) { | ||||||||
| 470 | ConstantSDNode *MaskN = dyn_cast<ConstantSDNode>(Node->getOperand(1)); | ||||||||
| 471 | if (!MaskN) | ||||||||
| 472 | return; | ||||||||
| 473 | |||||||||
| 474 | // The Reg operand should be a virtual register, which is defined | ||||||||
| 475 | // outside the current basic block. DAG combiner has done a pretty | ||||||||
| 476 | // good job in removing truncating inside a single basic block except | ||||||||
| 477 | // when the Reg operand comes from bpf_load_[byte | half | word] for | ||||||||
| 478 | // which the generic optimizer doesn't understand their results are | ||||||||
| 479 | // zero extended. | ||||||||
| 480 | SDValue BaseV = Node->getOperand(0); | ||||||||
| 481 | if (BaseV.getOpcode() != ISD::INTRINSIC_W_CHAIN) | ||||||||
| 482 | return; | ||||||||
| 483 | |||||||||
| 484 | unsigned IntNo = cast<ConstantSDNode>(BaseV->getOperand(1))->getZExtValue(); | ||||||||
| 485 | uint64_t MaskV = MaskN->getZExtValue(); | ||||||||
| 486 | |||||||||
| 487 | if (!((IntNo == Intrinsic::bpf_load_byte && MaskV == 0xFF) || | ||||||||
| 488 | (IntNo == Intrinsic::bpf_load_half && MaskV == 0xFFFF) || | ||||||||
| 489 | (IntNo == Intrinsic::bpf_load_word && MaskV == 0xFFFFFFFF))) | ||||||||
| 490 | return; | ||||||||
| 491 | |||||||||
| 492 | LLVM_DEBUG(dbgs() << "Remove the redundant AND operation in: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Remove the redundant AND operation in: " ; Node->dump(); dbgs() << '\n'; } } while (false) | ||||||||
| 493 | Node->dump(); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("bpf-isel")) { dbgs() << "Remove the redundant AND operation in: " ; Node->dump(); dbgs() << '\n'; } } while (false); | ||||||||
| 494 | |||||||||
| 495 | I--; | ||||||||
| 496 | CurDAG->ReplaceAllUsesWith(SDValue(Node, 0), BaseV); | ||||||||
| 497 | I++; | ||||||||
| 498 | CurDAG->DeleteNode(Node); | ||||||||
| 499 | } | ||||||||
| 500 | |||||||||
| 501 | FunctionPass *llvm::createBPFISelDag(BPFTargetMachine &TM) { | ||||||||
| 502 | return new BPFDAGToDAGISel(TM); | ||||||||
| 503 | } |
| 1 | //==- llvm/CodeGen/MachineMemOperand.h - MachineMemOperand class -*- C++ -*-==// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains the declaration of the MachineMemOperand class, which is a |
| 10 | // description of a memory reference. It is used to help track dependencies |
| 11 | // in the backend. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #ifndef LLVM_CODEGEN_MACHINEMEMOPERAND_H |
| 16 | #define LLVM_CODEGEN_MACHINEMEMOPERAND_H |
| 17 | |
| 18 | #include "llvm/ADT/BitmaskEnum.h" |
| 19 | #include "llvm/ADT/PointerUnion.h" |
| 20 | #include "llvm/CodeGen/PseudoSourceValue.h" |
| 21 | #include "llvm/IR/DerivedTypes.h" |
| 22 | #include "llvm/IR/Value.h" // PointerLikeTypeTraits<Value*> |
| 23 | #include "llvm/Support/AtomicOrdering.h" |
| 24 | #include "llvm/Support/DataTypes.h" |
| 25 | #include "llvm/Support/LowLevelTypeImpl.h" |
| 26 | |
| 27 | namespace llvm { |
| 28 | |
| 29 | class FoldingSetNodeID; |
| 30 | class MDNode; |
| 31 | class raw_ostream; |
| 32 | class MachineFunction; |
| 33 | class ModuleSlotTracker; |
| 34 | class TargetInstrInfo; |
| 35 | |
| 36 | /// This class contains a discriminated union of information about pointers in |
| 37 | /// memory operands, relating them back to LLVM IR or to virtual locations (such |
| 38 | /// as frame indices) that are exposed during codegen. |
| 39 | struct MachinePointerInfo { |
| 40 | /// This is the IR pointer value for the access, or it is null if unknown. |
| 41 | PointerUnion<const Value *, const PseudoSourceValue *> V; |
| 42 | |
| 43 | /// Offset - This is an offset from the base Value*. |
| 44 | int64_t Offset; |
| 45 | |
| 46 | unsigned AddrSpace = 0; |
| 47 | |
| 48 | uint8_t StackID; |
| 49 | |
| 50 | explicit MachinePointerInfo(const Value *v, int64_t offset = 0, |
| 51 | uint8_t ID = 0) |
| 52 | : V(v), Offset(offset), StackID(ID) { |
| 53 | AddrSpace = v ? v->getType()->getPointerAddressSpace() : 0; |
| 54 | } |
| 55 | |
| 56 | explicit MachinePointerInfo(const PseudoSourceValue *v, int64_t offset = 0, |
| 57 | uint8_t ID = 0) |
| 58 | : V(v), Offset(offset), StackID(ID) { |
| 59 | AddrSpace = v ? v->getAddressSpace() : 0; |
| 60 | } |
| 61 | |
| 62 | explicit MachinePointerInfo(unsigned AddressSpace = 0, int64_t offset = 0) |
| 63 | : V((const Value *)nullptr), Offset(offset), AddrSpace(AddressSpace), |
| 64 | StackID(0) {} |
| 65 | |
| 66 | explicit MachinePointerInfo( |
| 67 | PointerUnion<const Value *, const PseudoSourceValue *> v, |
| 68 | int64_t offset = 0, |
| 69 | uint8_t ID = 0) |
| 70 | : V(v), Offset(offset), StackID(ID) { |
| 71 | if (V) { |
| 72 | if (const auto *ValPtr = V.dyn_cast<const Value*>()) |
| 73 | AddrSpace = ValPtr->getType()->getPointerAddressSpace(); |
| 74 | else |
| 75 | AddrSpace = V.get<const PseudoSourceValue*>()->getAddressSpace(); |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | MachinePointerInfo getWithOffset(int64_t O) const { |
| 80 | if (V.isNull()) |
| 81 | return MachinePointerInfo(AddrSpace, Offset + O); |
| 82 | if (V.is<const Value*>()) |
| 83 | return MachinePointerInfo(V.get<const Value*>(), Offset + O, StackID); |
| 84 | return MachinePointerInfo(V.get<const PseudoSourceValue*>(), Offset + O, |
| 85 | StackID); |
| 86 | } |
| 87 | |
| 88 | /// Return true if memory region [V, V+Offset+Size) is known to be |
| 89 | /// dereferenceable. |
| 90 | bool isDereferenceable(unsigned Size, LLVMContext &C, |
| 91 | const DataLayout &DL) const; |
| 92 | |
| 93 | /// Return the LLVM IR address space number that this pointer points into. |
| 94 | unsigned getAddrSpace() const; |
| 95 | |
| 96 | /// Return a MachinePointerInfo record that refers to the constant pool. |
| 97 | static MachinePointerInfo getConstantPool(MachineFunction &MF); |
| 98 | |
| 99 | /// Return a MachinePointerInfo record that refers to the specified |
| 100 | /// FrameIndex. |
| 101 | static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, |
| 102 | int64_t Offset = 0); |
| 103 | |
| 104 | /// Return a MachinePointerInfo record that refers to a jump table entry. |
| 105 | static MachinePointerInfo getJumpTable(MachineFunction &MF); |
| 106 | |
| 107 | /// Return a MachinePointerInfo record that refers to a GOT entry. |
| 108 | static MachinePointerInfo getGOT(MachineFunction &MF); |
| 109 | |
| 110 | /// Stack pointer relative access. |
| 111 | static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, |
| 112 | uint8_t ID = 0); |
| 113 | |
| 114 | /// Stack memory without other information. |
| 115 | static MachinePointerInfo getUnknownStack(MachineFunction &MF); |
| 116 | }; |
| 117 | |
| 118 | |
| 119 | //===----------------------------------------------------------------------===// |
| 120 | /// A description of a memory reference used in the backend. |
| 121 | /// Instead of holding a StoreInst or LoadInst, this class holds the address |
| 122 | /// Value of the reference along with a byte size and offset. This allows it |
| 123 | /// to describe lowered loads and stores. Also, the special PseudoSourceValue |
| 124 | /// objects can be used to represent loads and stores to memory locations |
| 125 | /// that aren't explicit in the regular LLVM IR. |
| 126 | /// |
| 127 | class MachineMemOperand { |
| 128 | public: |
| 129 | /// Flags values. These may be or'd together. |
| 130 | enum Flags : uint16_t { |
| 131 | // No flags set. |
| 132 | MONone = 0, |
| 133 | /// The memory access reads data. |
| 134 | MOLoad = 1u << 0, |
| 135 | /// The memory access writes data. |
| 136 | MOStore = 1u << 1, |
| 137 | /// The memory access is volatile. |
| 138 | MOVolatile = 1u << 2, |
| 139 | /// The memory access is non-temporal. |
| 140 | MONonTemporal = 1u << 3, |
| 141 | /// The memory access is dereferenceable (i.e., doesn't trap). |
| 142 | MODereferenceable = 1u << 4, |
| 143 | /// The memory access always returns the same value (or traps). |
| 144 | MOInvariant = 1u << 5, |
| 145 | |
| 146 | // Reserved for use by target-specific passes. |
| 147 | // Targets may override getSerializableMachineMemOperandTargetFlags() to |
| 148 | // enable MIR serialization/parsing of these flags. If more of these flags |
| 149 | // are added, the MIR printing/parsing code will need to be updated as well. |
| 150 | MOTargetFlag1 = 1u << 6, |
| 151 | MOTargetFlag2 = 1u << 7, |
| 152 | MOTargetFlag3 = 1u << 8, |
| 153 | |
| 154 | LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ MOTargetFlag3)LLVM_BITMASK_LARGEST_ENUMERATOR = MOTargetFlag3 |
| 155 | }; |
| 156 | |
| 157 | private: |
| 158 | /// Atomic information for this memory operation. |
| 159 | struct MachineAtomicInfo { |
| 160 | /// Synchronization scope ID for this memory operation. |
| 161 | unsigned SSID : 8; // SyncScope::ID |
| 162 | /// Atomic ordering requirements for this memory operation. For cmpxchg |
| 163 | /// atomic operations, atomic ordering requirements when store occurs. |
| 164 | unsigned Ordering : 4; // enum AtomicOrdering |
| 165 | /// For cmpxchg atomic operations, atomic ordering requirements when store |
| 166 | /// does not occur. |
| 167 | unsigned FailureOrdering : 4; // enum AtomicOrdering |
| 168 | }; |
| 169 | |
| 170 | MachinePointerInfo PtrInfo; |
| 171 | |
| 172 | /// Track the memory type of the access. An access size which is unknown or |
| 173 | /// too large to be represented by LLT should use the invalid LLT. |
| 174 | LLT MemoryType; |
| 175 | |
| 176 | Flags FlagVals; |
| 177 | Align BaseAlign; |
| 178 | MachineAtomicInfo AtomicInfo; |
| 179 | AAMDNodes AAInfo; |
| 180 | const MDNode *Ranges; |
| 181 | |
| 182 | public: |
| 183 | /// Construct a MachineMemOperand object with the specified PtrInfo, flags, |
| 184 | /// size, and base alignment. For atomic operations the synchronization scope |
| 185 | /// and atomic ordering requirements must also be specified. For cmpxchg |
| 186 | /// atomic operations the atomic ordering requirements when store does not |
| 187 | /// occur must also be specified. |
| 188 | MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, uint64_t s, |
| 189 | Align a, const AAMDNodes &AAInfo = AAMDNodes(), |
| 190 | const MDNode *Ranges = nullptr, |
| 191 | SyncScope::ID SSID = SyncScope::System, |
| 192 | AtomicOrdering Ordering = AtomicOrdering::NotAtomic, |
| 193 | AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); |
| 194 | MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a, |
| 195 | const AAMDNodes &AAInfo = AAMDNodes(), |
| 196 | const MDNode *Ranges = nullptr, |
| 197 | SyncScope::ID SSID = SyncScope::System, |
| 198 | AtomicOrdering Ordering = AtomicOrdering::NotAtomic, |
| 199 | AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic); |
| 200 | |
| 201 | const MachinePointerInfo &getPointerInfo() const { return PtrInfo; } |
| 202 | |
| 203 | /// Return the base address of the memory access. This may either be a normal |
| 204 | /// LLVM IR Value, or one of the special values used in CodeGen. |
| 205 | /// Special values are those obtained via |
| 206 | /// PseudoSourceValue::getFixedStack(int), PseudoSourceValue::getStack, and |
| 207 | /// other PseudoSourceValue member functions which return objects which stand |
| 208 | /// for frame/stack pointer relative references and other special references |
| 209 | /// which are not representable in the high-level IR. |
| 210 | const Value *getValue() const { return PtrInfo.V.dyn_cast<const Value*>(); } |
| 211 | |
| 212 | const PseudoSourceValue *getPseudoValue() const { |
| 213 | return PtrInfo.V.dyn_cast<const PseudoSourceValue*>(); |
| 214 | } |
| 215 | |
| 216 | const void *getOpaqueValue() const { return PtrInfo.V.getOpaqueValue(); } |
| 217 | |
| 218 | /// Return the raw flags of the source value, \see Flags. |
| 219 | Flags getFlags() const { return FlagVals; } |
| 220 | |
| 221 | /// Bitwise OR the current flags with the given flags. |
| 222 | void setFlags(Flags f) { FlagVals |= f; } |
| 223 | |
| 224 | /// For normal values, this is a byte offset added to the base address. |
| 225 | /// For PseudoSourceValue::FPRel values, this is the FrameIndex number. |
| 226 | int64_t getOffset() const { return PtrInfo.Offset; } |
| 227 | |
| 228 | unsigned getAddrSpace() const { return PtrInfo.getAddrSpace(); } |
| 229 | |
| 230 | /// Return the memory type of the memory reference. This should only be relied |
| 231 | /// on for GlobalISel G_* operation legalization. |
| 232 | LLT getMemoryType() const { return MemoryType; } |
| 233 | |
| 234 | /// Return the size in bytes of the memory reference. |
| 235 | uint64_t getSize() const { |
| 236 | return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0)0UL; |
| 237 | } |
| 238 | |
| 239 | /// Return the size in bits of the memory reference. |
| 240 | uint64_t getSizeInBits() const { |
| 241 | return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0)0UL; |
| 242 | } |
| 243 | |
| 244 | LLT getType() const { |
| 245 | return MemoryType; |
| 246 | } |
| 247 | |
| 248 | /// Return the minimum known alignment in bytes of the actual memory |
| 249 | /// reference. |
| 250 | Align getAlign() const; |
| 251 | |
| 252 | /// Return the minimum known alignment in bytes of the base address, without |
| 253 | /// the offset. |
| 254 | Align getBaseAlign() const { return BaseAlign; } |
| 255 | |
| 256 | /// Return the AA tags for the memory reference. |
| 257 | AAMDNodes getAAInfo() const { return AAInfo; } |
| 258 | |
| 259 | /// Return the range tag for the memory reference. |
| 260 | const MDNode *getRanges() const { return Ranges; } |
| 261 | |
| 262 | /// Returns the synchronization scope ID for this memory operation. |
| 263 | SyncScope::ID getSyncScopeID() const { |
| 264 | return static_cast<SyncScope::ID>(AtomicInfo.SSID); |
| 265 | } |
| 266 | |
| 267 | /// Return the atomic ordering requirements for this memory operation. For |
| 268 | /// cmpxchg atomic operations, return the atomic ordering requirements when |
| 269 | /// store occurs. |
| 270 | AtomicOrdering getSuccessOrdering() const { |
| 271 | return static_cast<AtomicOrdering>(AtomicInfo.Ordering); |
| 272 | } |
| 273 | |
| 274 | /// For cmpxchg atomic operations, return the atomic ordering requirements |
| 275 | /// when store does not occur. |
| 276 | AtomicOrdering getFailureOrdering() const { |
| 277 | return static_cast<AtomicOrdering>(AtomicInfo.FailureOrdering); |
| 278 | } |
| 279 | |
| 280 | /// Return a single atomic ordering that is at least as strong as both the |
| 281 | /// success and failure orderings for an atomic operation. (For operations |
| 282 | /// other than cmpxchg, this is equivalent to getSuccessOrdering().) |
| 283 | AtomicOrdering getMergedOrdering() const { |
| 284 | return getMergedAtomicOrdering(getSuccessOrdering(), getFailureOrdering()); |
| 285 | } |
| 286 | |
| 287 | bool isLoad() const { return FlagVals & MOLoad; } |
| 288 | bool isStore() const { return FlagVals & MOStore; } |
| 289 | bool isVolatile() const { return FlagVals & MOVolatile; } |
| 290 | bool isNonTemporal() const { return FlagVals & MONonTemporal; } |
| 291 | bool isDereferenceable() const { return FlagVals & MODereferenceable; } |
| 292 | bool isInvariant() const { return FlagVals & MOInvariant; } |
| 293 | |
| 294 | /// Returns true if this operation has an atomic ordering requirement of |
| 295 | /// unordered or higher, false otherwise. |
| 296 | bool isAtomic() const { |
| 297 | return getSuccessOrdering() != AtomicOrdering::NotAtomic; |
| 298 | } |
| 299 | |
| 300 | /// Returns true if this memory operation doesn't have any ordering |
| 301 | /// constraints other than normal aliasing. Volatile and (ordered) atomic |
| 302 | /// memory operations can't be reordered. |
| 303 | bool isUnordered() const { |
| 304 | return (getSuccessOrdering() == AtomicOrdering::NotAtomic || |
| 305 | getSuccessOrdering() == AtomicOrdering::Unordered) && |
| 306 | !isVolatile(); |
| 307 | } |
| 308 | |
| 309 | /// Update this MachineMemOperand to reflect the alignment of MMO, if it has a |
| 310 | /// greater alignment. This must only be used when the new alignment applies |
| 311 | /// to all users of this MachineMemOperand. |
| 312 | void refineAlignment(const MachineMemOperand *MMO); |
| 313 | |
| 314 | /// Change the SourceValue for this MachineMemOperand. This should only be |
| 315 | /// used when an object is being relocated and all references to it are being |
| 316 | /// updated. |
| 317 | void setValue(const Value *NewSV) { PtrInfo.V = NewSV; } |
| 318 | void setValue(const PseudoSourceValue *NewSV) { PtrInfo.V = NewSV; } |
| 319 | void setOffset(int64_t NewOffset) { PtrInfo.Offset = NewOffset; } |
| 320 | |
| 321 | /// Reset the tracked memory type. |
| 322 | void setType(LLT NewTy) { |
| 323 | MemoryType = NewTy; |
| 324 | } |
| 325 | |
| 326 | /// Profile - Gather unique data for the object. |
| 327 | /// |
| 328 | void Profile(FoldingSetNodeID &ID) const; |
| 329 | |
| 330 | /// Support for operator<<. |
| 331 | /// @{ |
| 332 | void print(raw_ostream &OS, ModuleSlotTracker &MST, |
| 333 | SmallVectorImpl<StringRef> &SSNs, const LLVMContext &Context, |
| 334 | const MachineFrameInfo *MFI, const TargetInstrInfo *TII) const; |
| 335 | /// @} |
| 336 | |
| 337 | friend bool operator==(const MachineMemOperand &LHS, |
| 338 | const MachineMemOperand &RHS) { |
| 339 | return LHS.getValue() == RHS.getValue() && |
| 340 | LHS.getPseudoValue() == RHS.getPseudoValue() && |
| 341 | LHS.getSize() == RHS.getSize() && |
| 342 | LHS.getOffset() == RHS.getOffset() && |
| 343 | LHS.getFlags() == RHS.getFlags() && |
| 344 | LHS.getAAInfo() == RHS.getAAInfo() && |
| 345 | LHS.getRanges() == RHS.getRanges() && |
| 346 | LHS.getAlign() == RHS.getAlign() && |
| 347 | LHS.getAddrSpace() == RHS.getAddrSpace(); |
| 348 | } |
| 349 | |
| 350 | friend bool operator!=(const MachineMemOperand &LHS, |
| 351 | const MachineMemOperand &RHS) { |
| 352 | return !(LHS == RHS); |
| 353 | } |
| 354 | }; |
| 355 | |
| 356 | } // End llvm namespace |
| 357 | |
| 358 | #endif |