| File: | build/source/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp |
| Warning: | line 223, column 9 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- OpenMPToLLVMIRTranslation.cpp - Translate OpenMP dialect to LLVM IR-===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // This file implements a translation between the MLIR OpenMP dialect and LLVM | |||
| 10 | // IR. | |||
| 11 | // | |||
| 12 | //===----------------------------------------------------------------------===// | |||
| 13 | #include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h" | |||
| 14 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" | |||
| 15 | #include "mlir/Dialect/OpenMP/OpenMPInterfaces.h" | |||
| 16 | #include "mlir/IR/IRMapping.h" | |||
| 17 | #include "mlir/IR/Operation.h" | |||
| 18 | #include "mlir/Support/LLVM.h" | |||
| 19 | #include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h" | |||
| 20 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" | |||
| 21 | #include "mlir/Transforms/RegionUtils.h" | |||
| 22 | ||||
| 23 | #include "llvm/ADT/SetVector.h" | |||
| 24 | #include "llvm/ADT/TypeSwitch.h" | |||
| 25 | #include "llvm/Frontend/OpenMP/OMPConstants.h" | |||
| 26 | #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" | |||
| 27 | #include "llvm/IR/DebugInfoMetadata.h" | |||
| 28 | #include "llvm/IR/IRBuilder.h" | |||
| 29 | #include "llvm/Support/FileSystem.h" | |||
| 30 | ||||
| 31 | using namespace mlir; | |||
| 32 | ||||
| 33 | namespace { | |||
| 34 | static llvm::omp::ScheduleKind | |||
| 35 | convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) { | |||
| 36 | if (!schedKind.has_value()) | |||
| 37 | return llvm::omp::OMP_SCHEDULE_Default; | |||
| 38 | switch (schedKind.value()) { | |||
| 39 | case omp::ClauseScheduleKind::Static: | |||
| 40 | return llvm::omp::OMP_SCHEDULE_Static; | |||
| 41 | case omp::ClauseScheduleKind::Dynamic: | |||
| 42 | return llvm::omp::OMP_SCHEDULE_Dynamic; | |||
| 43 | case omp::ClauseScheduleKind::Guided: | |||
| 44 | return llvm::omp::OMP_SCHEDULE_Guided; | |||
| 45 | case omp::ClauseScheduleKind::Auto: | |||
| 46 | return llvm::omp::OMP_SCHEDULE_Auto; | |||
| 47 | case omp::ClauseScheduleKind::Runtime: | |||
| 48 | return llvm::omp::OMP_SCHEDULE_Runtime; | |||
| 49 | } | |||
| 50 | llvm_unreachable("unhandled schedule clause argument")::llvm::llvm_unreachable_internal("unhandled schedule clause argument" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 50); | |||
| 51 | } | |||
| 52 | ||||
| 53 | /// ModuleTranslation stack frame for OpenMP operations. This keeps track of the | |||
| 54 | /// insertion points for allocas. | |||
| 55 | class OpenMPAllocaStackFrame | |||
| 56 | : public LLVM::ModuleTranslation::StackFrameBase<OpenMPAllocaStackFrame> { | |||
| 57 | public: | |||
| 58 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocaStackFrame)static ::mlir::TypeID resolveTypeID() { static ::mlir::SelfOwningTypeID id; return id; } static_assert( ::mlir::detail::InlineTypeIDResolver ::has_resolve_typeid< OpenMPAllocaStackFrame>::value, "`MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID` must be placed in a " "public section of `" "OpenMPAllocaStackFrame" "`"); | |||
| 59 | ||||
| 60 | explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP) | |||
| 61 | : allocaInsertPoint(allocaIP) {} | |||
| 62 | llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint; | |||
| 63 | }; | |||
| 64 | ||||
| 65 | /// ModuleTranslation stack frame containing the partial mapping between MLIR | |||
| 66 | /// values and their LLVM IR equivalents. | |||
| 67 | class OpenMPVarMappingStackFrame | |||
| 68 | : public LLVM::ModuleTranslation::StackFrameBase< | |||
| 69 | OpenMPVarMappingStackFrame> { | |||
| 70 | public: | |||
| 71 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPVarMappingStackFrame)static ::mlir::TypeID resolveTypeID() { static ::mlir::SelfOwningTypeID id; return id; } static_assert( ::mlir::detail::InlineTypeIDResolver ::has_resolve_typeid< OpenMPVarMappingStackFrame>::value , "`MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID` must be placed in a " "public section of `" "OpenMPVarMappingStackFrame" "`"); | |||
| 72 | ||||
| 73 | explicit OpenMPVarMappingStackFrame( | |||
| 74 | const DenseMap<Value, llvm::Value *> &mapping) | |||
| 75 | : mapping(mapping) {} | |||
| 76 | ||||
| 77 | DenseMap<Value, llvm::Value *> mapping; | |||
| 78 | }; | |||
| 79 | } // namespace | |||
| 80 | ||||
| 81 | /// Find the insertion point for allocas given the current insertion point for | |||
| 82 | /// normal operations in the builder. | |||
| 83 | static llvm::OpenMPIRBuilder::InsertPointTy | |||
| 84 | findAllocaInsertPoint(llvm::IRBuilderBase &builder, | |||
| 85 | const LLVM::ModuleTranslation &moduleTranslation) { | |||
| 86 | // If there is an alloca insertion point on stack, i.e. we are in a nested | |||
| 87 | // operation and a specific point was provided by some surrounding operation, | |||
| 88 | // use it. | |||
| 89 | llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint; | |||
| 90 | WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocaStackFrame>( | |||
| 91 | [&](const OpenMPAllocaStackFrame &frame) { | |||
| 92 | allocaInsertPoint = frame.allocaInsertPoint; | |||
| 93 | return WalkResult::interrupt(); | |||
| 94 | }); | |||
| 95 | if (walkResult.wasInterrupted()) | |||
| 96 | return allocaInsertPoint; | |||
| 97 | ||||
| 98 | // Otherwise, insert to the entry block of the surrounding function. | |||
| 99 | // If the current IRBuilder InsertPoint is the function's entry, it cannot | |||
| 100 | // also be used for alloca insertion which would result in insertion order | |||
| 101 | // confusion. Create a new BasicBlock for the Builder and use the entry block | |||
| 102 | // for the allocs. | |||
| 103 | // TODO: Create a dedicated alloca BasicBlock at function creation such that | |||
| 104 | // we do not need to move the current InertPoint here. | |||
| 105 | if (builder.GetInsertBlock() == | |||
| 106 | &builder.GetInsertBlock()->getParent()->getEntryBlock()) { | |||
| 107 | assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&(static_cast <bool> (builder.GetInsertPoint() == builder .GetInsertBlock()->end() && "Assuming end of basic block" ) ? void (0) : __assert_fail ("builder.GetInsertPoint() == builder.GetInsertBlock()->end() && \"Assuming end of basic block\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 108, __extension__ __PRETTY_FUNCTION__)) | |||
| 108 | "Assuming end of basic block")(static_cast <bool> (builder.GetInsertPoint() == builder .GetInsertBlock()->end() && "Assuming end of basic block" ) ? void (0) : __assert_fail ("builder.GetInsertPoint() == builder.GetInsertBlock()->end() && \"Assuming end of basic block\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 108, __extension__ __PRETTY_FUNCTION__)); | |||
| 109 | llvm::BasicBlock *entryBB = llvm::BasicBlock::Create( | |||
| 110 | builder.getContext(), "entry", builder.GetInsertBlock()->getParent(), | |||
| 111 | builder.GetInsertBlock()->getNextNode()); | |||
| 112 | builder.CreateBr(entryBB); | |||
| 113 | builder.SetInsertPoint(entryBB); | |||
| 114 | } | |||
| 115 | ||||
| 116 | llvm::BasicBlock &funcEntryBlock = | |||
| 117 | builder.GetInsertBlock()->getParent()->getEntryBlock(); | |||
| 118 | return llvm::OpenMPIRBuilder::InsertPointTy( | |||
| 119 | &funcEntryBlock, funcEntryBlock.getFirstInsertionPt()); | |||
| 120 | } | |||
| 121 | ||||
| 122 | /// Converts the given region that appears within an OpenMP dialect operation to | |||
| 123 | /// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the | |||
| 124 | /// region, and a branch from any block with an successor-less OpenMP terminator | |||
| 125 | /// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes | |||
| 126 | /// of the continuation block if provided. | |||
| 127 | static llvm::BasicBlock *convertOmpOpRegions( | |||
| 128 | Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, | |||
| 129 | LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus, | |||
| 130 | SmallVectorImpl<llvm::PHINode *> *continuationBlockPHIs = nullptr) { | |||
| 131 | llvm::BasicBlock *continuationBlock = | |||
| 132 | splitBB(builder, true, "omp.region.cont"); | |||
| 133 | llvm::BasicBlock *sourceBlock = builder.GetInsertBlock(); | |||
| 134 | ||||
| 135 | llvm::LLVMContext &llvmContext = builder.getContext(); | |||
| 136 | for (Block &bb : region) { | |||
| 137 | llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create( | |||
| 138 | llvmContext, blockName, builder.GetInsertBlock()->getParent(), | |||
| 139 | builder.GetInsertBlock()->getNextNode()); | |||
| 140 | moduleTranslation.mapBlock(&bb, llvmBB); | |||
| 141 | } | |||
| 142 | ||||
| 143 | llvm::Instruction *sourceTerminator = sourceBlock->getTerminator(); | |||
| 144 | ||||
| 145 | // Terminators (namely YieldOp) may be forwarding values to the region that | |||
| 146 | // need to be available in the continuation block. Collect the types of these | |||
| 147 | // operands in preparation of creating PHI nodes. | |||
| 148 | SmallVector<llvm::Type *> continuationBlockPHITypes; | |||
| 149 | bool operandsProcessed = false; | |||
| 150 | unsigned numYields = 0; | |||
| 151 | for (Block &bb : region.getBlocks()) { | |||
| 152 | if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) { | |||
| 153 | if (!operandsProcessed) { | |||
| 154 | for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) { | |||
| 155 | continuationBlockPHITypes.push_back( | |||
| 156 | moduleTranslation.convertType(yield->getOperand(i).getType())); | |||
| 157 | } | |||
| 158 | operandsProcessed = true; | |||
| 159 | } else { | |||
| 160 | assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&(static_cast <bool> (continuationBlockPHITypes.size() == yield->getNumOperands() && "mismatching number of values yielded from the region" ) ? void (0) : __assert_fail ("continuationBlockPHITypes.size() == yield->getNumOperands() && \"mismatching number of values yielded from the region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 161, __extension__ __PRETTY_FUNCTION__)) | |||
| 161 | "mismatching number of values yielded from the region")(static_cast <bool> (continuationBlockPHITypes.size() == yield->getNumOperands() && "mismatching number of values yielded from the region" ) ? void (0) : __assert_fail ("continuationBlockPHITypes.size() == yield->getNumOperands() && \"mismatching number of values yielded from the region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 161, __extension__ __PRETTY_FUNCTION__)); | |||
| 162 | for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) { | |||
| 163 | llvm::Type *operandType = | |||
| 164 | moduleTranslation.convertType(yield->getOperand(i).getType()); | |||
| 165 | (void)operandType; | |||
| 166 | assert(continuationBlockPHITypes[i] == operandType &&(static_cast <bool> (continuationBlockPHITypes[i] == operandType && "values of mismatching types yielded from the region" ) ? void (0) : __assert_fail ("continuationBlockPHITypes[i] == operandType && \"values of mismatching types yielded from the region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 167, __extension__ __PRETTY_FUNCTION__)) | |||
| 167 | "values of mismatching types yielded from the region")(static_cast <bool> (continuationBlockPHITypes[i] == operandType && "values of mismatching types yielded from the region" ) ? void (0) : __assert_fail ("continuationBlockPHITypes[i] == operandType && \"values of mismatching types yielded from the region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 167, __extension__ __PRETTY_FUNCTION__)); | |||
| 168 | } | |||
| 169 | } | |||
| 170 | numYields++; | |||
| 171 | } | |||
| 172 | } | |||
| 173 | ||||
| 174 | // Insert PHI nodes in the continuation block for any values forwarded by the | |||
| 175 | // terminators in this region. | |||
| 176 | if (!continuationBlockPHITypes.empty()) | |||
| 177 | assert((static_cast <bool> (continuationBlockPHIs && "expected continuation block PHIs if converted regions yield values" ) ? void (0) : __assert_fail ("continuationBlockPHIs && \"expected continuation block PHIs if converted regions yield values\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 179, __extension__ __PRETTY_FUNCTION__)) | |||
| 178 | continuationBlockPHIs &&(static_cast <bool> (continuationBlockPHIs && "expected continuation block PHIs if converted regions yield values" ) ? void (0) : __assert_fail ("continuationBlockPHIs && \"expected continuation block PHIs if converted regions yield values\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 179, __extension__ __PRETTY_FUNCTION__)) | |||
| 179 | "expected continuation block PHIs if converted regions yield values")(static_cast <bool> (continuationBlockPHIs && "expected continuation block PHIs if converted regions yield values" ) ? void (0) : __assert_fail ("continuationBlockPHIs && \"expected continuation block PHIs if converted regions yield values\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 179, __extension__ __PRETTY_FUNCTION__)); | |||
| 180 | if (continuationBlockPHIs
| |||
| 181 | llvm::IRBuilderBase::InsertPointGuard guard(builder); | |||
| 182 | continuationBlockPHIs->reserve(continuationBlockPHITypes.size()); | |||
| 183 | builder.SetInsertPoint(continuationBlock, continuationBlock->begin()); | |||
| 184 | for (llvm::Type *ty : continuationBlockPHITypes) | |||
| 185 | continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields)); | |||
| 186 | } | |||
| 187 | ||||
| 188 | // Convert blocks one by one in topological order to ensure | |||
| 189 | // defs are converted before uses. | |||
| 190 | SetVector<Block *> blocks = | |||
| 191 | LLVM::detail::getTopologicallySortedBlocks(region); | |||
| 192 | for (Block *bb : blocks) { | |||
| 193 | llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb); | |||
| 194 | // Retarget the branch of the entry block to the entry block of the | |||
| 195 | // converted region (regions are single-entry). | |||
| 196 | if (bb->isEntryBlock()) { | |||
| 197 | assert(sourceTerminator->getNumSuccessors() == 1 &&(static_cast <bool> (sourceTerminator->getNumSuccessors () == 1 && "provided entry block has multiple successors" ) ? void (0) : __assert_fail ("sourceTerminator->getNumSuccessors() == 1 && \"provided entry block has multiple successors\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 198, __extension__ __PRETTY_FUNCTION__)) | |||
| 198 | "provided entry block has multiple successors")(static_cast <bool> (sourceTerminator->getNumSuccessors () == 1 && "provided entry block has multiple successors" ) ? void (0) : __assert_fail ("sourceTerminator->getNumSuccessors() == 1 && \"provided entry block has multiple successors\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 198, __extension__ __PRETTY_FUNCTION__)); | |||
| 199 | assert(sourceTerminator->getSuccessor(0) == continuationBlock &&(static_cast <bool> (sourceTerminator->getSuccessor( 0) == continuationBlock && "ContinuationBlock is not the successor of the entry block" ) ? void (0) : __assert_fail ("sourceTerminator->getSuccessor(0) == continuationBlock && \"ContinuationBlock is not the successor of the entry block\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 200, __extension__ __PRETTY_FUNCTION__)) | |||
| 200 | "ContinuationBlock is not the successor of the entry block")(static_cast <bool> (sourceTerminator->getSuccessor( 0) == continuationBlock && "ContinuationBlock is not the successor of the entry block" ) ? void (0) : __assert_fail ("sourceTerminator->getSuccessor(0) == continuationBlock && \"ContinuationBlock is not the successor of the entry block\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 200, __extension__ __PRETTY_FUNCTION__)); | |||
| 201 | sourceTerminator->setSuccessor(0, llvmBB); | |||
| 202 | } | |||
| 203 | ||||
| 204 | llvm::IRBuilderBase::InsertPointGuard guard(builder); | |||
| 205 | if (failed( | |||
| 206 | moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder))) { | |||
| 207 | bodyGenStatus = failure(); | |||
| 208 | return continuationBlock; | |||
| 209 | } | |||
| 210 | ||||
| 211 | // Special handling for `omp.yield` and `omp.terminator` (we may have more | |||
| 212 | // than one): they return the control to the parent OpenMP dialect operation | |||
| 213 | // so replace them with the branch to the continuation block. We handle this | |||
| 214 | // here to avoid relying inter-function communication through the | |||
| 215 | // ModuleTranslation class to set up the correct insertion point. This is | |||
| 216 | // also consistent with MLIR's idiom of handling special region terminators | |||
| 217 | // in the same code that handles the region-owning operation. | |||
| 218 | Operation *terminator = bb->getTerminator(); | |||
| 219 | if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) { | |||
| 220 | builder.CreateBr(continuationBlock); | |||
| 221 | ||||
| 222 | for (unsigned i = 0, e = terminator->getNumOperands(); i < e; ++i) | |||
| 223 | (*continuationBlockPHIs)[i]->addIncoming( | |||
| ||||
| 224 | moduleTranslation.lookupValue(terminator->getOperand(i)), llvmBB); | |||
| 225 | } | |||
| 226 | } | |||
| 227 | // After all blocks have been traversed and values mapped, connect the PHI | |||
| 228 | // nodes to the results of preceding blocks. | |||
| 229 | LLVM::detail::connectPHINodes(region, moduleTranslation); | |||
| 230 | ||||
| 231 | // Remove the blocks and values defined in this region from the mapping since | |||
| 232 | // they are not visible outside of this region. This allows the same region to | |||
| 233 | // be converted several times, that is cloned, without clashes, and slightly | |||
| 234 | // speeds up the lookups. | |||
| 235 | moduleTranslation.forgetMapping(region); | |||
| 236 | ||||
| 237 | return continuationBlock; | |||
| 238 | } | |||
| 239 | ||||
| 240 | /// Convert ProcBindKind from MLIR-generated enum to LLVM enum. | |||
| 241 | static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind) { | |||
| 242 | switch (kind) { | |||
| 243 | case omp::ClauseProcBindKind::Close: | |||
| 244 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_close; | |||
| 245 | case omp::ClauseProcBindKind::Master: | |||
| 246 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_master; | |||
| 247 | case omp::ClauseProcBindKind::Primary: | |||
| 248 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary; | |||
| 249 | case omp::ClauseProcBindKind::Spread: | |||
| 250 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread; | |||
| 251 | } | |||
| 252 | llvm_unreachable("Unknown ClauseProcBindKind kind")::llvm::llvm_unreachable_internal("Unknown ClauseProcBindKind kind" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 252); | |||
| 253 | } | |||
| 254 | ||||
| 255 | /// Converts the OpenMP parallel operation to LLVM IR. | |||
| 256 | static LogicalResult | |||
| 257 | convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, | |||
| 258 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 259 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 260 | // TODO: support error propagation in OpenMPIRBuilder and use it instead of | |||
| 261 | // relying on captured variables. | |||
| 262 | LogicalResult bodyGenStatus = success(); | |||
| 263 | ||||
| 264 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 265 | // Save the alloca insertion point on ModuleTranslation stack for use in | |||
| 266 | // nested regions. | |||
| 267 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame( | |||
| 268 | moduleTranslation, allocaIP); | |||
| 269 | ||||
| 270 | // ParallelOp has only one region associated with it. | |||
| 271 | builder.restoreIP(codeGenIP); | |||
| 272 | convertOmpOpRegions(opInst.getRegion(), "omp.par.region", builder, | |||
| 273 | moduleTranslation, bodyGenStatus); | |||
| 274 | }; | |||
| 275 | ||||
| 276 | // TODO: Perform appropriate actions according to the data-sharing | |||
| 277 | // attribute (shared, private, firstprivate, ...) of variables. | |||
| 278 | // Currently defaults to shared. | |||
| 279 | auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, | |||
| 280 | llvm::Value &, llvm::Value &vPtr, | |||
| 281 | llvm::Value *&replacementValue) -> InsertPointTy { | |||
| 282 | replacementValue = &vPtr; | |||
| 283 | ||||
| 284 | return codeGenIP; | |||
| 285 | }; | |||
| 286 | ||||
| 287 | // TODO: Perform finalization actions for variables. This has to be | |||
| 288 | // called for variables which have destructors/finalizers. | |||
| 289 | auto finiCB = [&](InsertPointTy codeGenIP) {}; | |||
| 290 | ||||
| 291 | llvm::Value *ifCond = nullptr; | |||
| 292 | if (auto ifExprVar = opInst.getIfExprVar()) | |||
| 293 | ifCond = moduleTranslation.lookupValue(ifExprVar); | |||
| 294 | llvm::Value *numThreads = nullptr; | |||
| 295 | if (auto numThreadsVar = opInst.getNumThreadsVar()) | |||
| 296 | numThreads = moduleTranslation.lookupValue(numThreadsVar); | |||
| 297 | auto pbKind = llvm::omp::OMP_PROC_BIND_default; | |||
| 298 | if (auto bind = opInst.getProcBindVal()) | |||
| 299 | pbKind = getProcBindKind(*bind); | |||
| 300 | // TODO: Is the Parallel construct cancellable? | |||
| 301 | bool isCancellable = false; | |||
| 302 | ||||
| 303 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = | |||
| 304 | findAllocaInsertPoint(builder, moduleTranslation); | |||
| 305 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 306 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createParallel( | |||
| 307 | ompLoc, allocaIP, bodyGenCB, privCB, finiCB, ifCond, numThreads, pbKind, | |||
| 308 | isCancellable)); | |||
| 309 | ||||
| 310 | return bodyGenStatus; | |||
| 311 | } | |||
| 312 | ||||
| 313 | /// Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder. | |||
| 314 | static LogicalResult | |||
| 315 | convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 316 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 317 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 318 | // TODO: support error propagation in OpenMPIRBuilder and use it instead of | |||
| 319 | // relying on captured variables. | |||
| 320 | LogicalResult bodyGenStatus = success(); | |||
| 321 | ||||
| 322 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 323 | // MasterOp has only one region associated with it. | |||
| 324 | auto ®ion = cast<omp::MasterOp>(opInst).getRegion(); | |||
| 325 | builder.restoreIP(codeGenIP); | |||
| 326 | convertOmpOpRegions(region, "omp.master.region", builder, moduleTranslation, | |||
| 327 | bodyGenStatus); | |||
| 328 | }; | |||
| 329 | ||||
| 330 | // TODO: Perform finalization actions for variables. This has to be | |||
| 331 | // called for variables which have destructors/finalizers. | |||
| 332 | auto finiCB = [&](InsertPointTy codeGenIP) {}; | |||
| 333 | ||||
| 334 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 335 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createMaster( | |||
| 336 | ompLoc, bodyGenCB, finiCB)); | |||
| 337 | return success(); | |||
| 338 | } | |||
| 339 | ||||
| 340 | /// Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder. | |||
| 341 | static LogicalResult | |||
| 342 | convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 343 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 344 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 345 | auto criticalOp = cast<omp::CriticalOp>(opInst); | |||
| 346 | // TODO: support error propagation in OpenMPIRBuilder and use it instead of | |||
| 347 | // relying on captured variables. | |||
| 348 | LogicalResult bodyGenStatus = success(); | |||
| 349 | ||||
| 350 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 351 | // CriticalOp has only one region associated with it. | |||
| 352 | auto ®ion = cast<omp::CriticalOp>(opInst).getRegion(); | |||
| 353 | builder.restoreIP(codeGenIP); | |||
| 354 | convertOmpOpRegions(region, "omp.critical.region", builder, | |||
| 355 | moduleTranslation, bodyGenStatus); | |||
| 356 | }; | |||
| 357 | ||||
| 358 | // TODO: Perform finalization actions for variables. This has to be | |||
| 359 | // called for variables which have destructors/finalizers. | |||
| 360 | auto finiCB = [&](InsertPointTy codeGenIP) {}; | |||
| 361 | ||||
| 362 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 363 | llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); | |||
| 364 | llvm::Constant *hint = nullptr; | |||
| 365 | ||||
| 366 | // If it has a name, it probably has a hint too. | |||
| 367 | if (criticalOp.getNameAttr()) { | |||
| 368 | // The verifiers in OpenMP Dialect guarentee that all the pointers are | |||
| 369 | // non-null | |||
| 370 | auto symbolRef = criticalOp.getNameAttr().cast<SymbolRefAttr>(); | |||
| 371 | auto criticalDeclareOp = | |||
| 372 | SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp, | |||
| 373 | symbolRef); | |||
| 374 | hint = llvm::ConstantInt::get( | |||
| 375 | llvm::Type::getInt32Ty(llvmContext), | |||
| 376 | static_cast<int>(criticalDeclareOp.getHintVal())); | |||
| 377 | } | |||
| 378 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createCritical( | |||
| 379 | ompLoc, bodyGenCB, finiCB, criticalOp.getName().value_or(""), hint)); | |||
| 380 | return success(); | |||
| 381 | } | |||
| 382 | ||||
| 383 | /// Returns a reduction declaration that corresponds to the given reduction | |||
| 384 | /// operation in the given container. Currently only supports reductions inside | |||
| 385 | /// WsLoopOp but can be easily extended. | |||
| 386 | static omp::ReductionDeclareOp findReductionDecl(omp::WsLoopOp container, | |||
| 387 | omp::ReductionOp reduction) { | |||
| 388 | SymbolRefAttr reductionSymbol; | |||
| 389 | for (unsigned i = 0, e = container.getNumReductionVars(); i < e; ++i) { | |||
| 390 | if (container.getReductionVars()[i] != reduction.getAccumulator()) | |||
| 391 | continue; | |||
| 392 | reductionSymbol = (*container.getReductions())[i].cast<SymbolRefAttr>(); | |||
| 393 | break; | |||
| 394 | } | |||
| 395 | assert(reductionSymbol &&(static_cast <bool> (reductionSymbol && "reduction operation must be associated with a declaration" ) ? void (0) : __assert_fail ("reductionSymbol && \"reduction operation must be associated with a declaration\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 396, __extension__ __PRETTY_FUNCTION__)) | |||
| 396 | "reduction operation must be associated with a declaration")(static_cast <bool> (reductionSymbol && "reduction operation must be associated with a declaration" ) ? void (0) : __assert_fail ("reductionSymbol && \"reduction operation must be associated with a declaration\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 396, __extension__ __PRETTY_FUNCTION__)); | |||
| 397 | ||||
| 398 | return SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>( | |||
| 399 | container, reductionSymbol); | |||
| 400 | } | |||
| 401 | ||||
| 402 | /// Populates `reductions` with reduction declarations used in the given loop. | |||
| 403 | static void | |||
| 404 | collectReductionDecls(omp::WsLoopOp loop, | |||
| 405 | SmallVectorImpl<omp::ReductionDeclareOp> &reductions) { | |||
| 406 | std::optional<ArrayAttr> attr = loop.getReductions(); | |||
| 407 | if (!attr) | |||
| 408 | return; | |||
| 409 | ||||
| 410 | reductions.reserve(reductions.size() + loop.getNumReductionVars()); | |||
| 411 | for (auto symbolRef : attr->getAsRange<SymbolRefAttr>()) { | |||
| 412 | reductions.push_back( | |||
| 413 | SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>( | |||
| 414 | loop, symbolRef)); | |||
| 415 | } | |||
| 416 | } | |||
| 417 | ||||
| 418 | /// Translates the blocks contained in the given region and appends them to at | |||
| 419 | /// the current insertion point of `builder`. The operations of the entry block | |||
| 420 | /// are appended to the current insertion block, which is not expected to have a | |||
| 421 | /// terminator. If set, `continuationBlockArgs` is populated with translated | |||
| 422 | /// values that correspond to the values omp.yield'ed from the region. | |||
| 423 | static LogicalResult inlineConvertOmpRegions( | |||
| 424 | Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, | |||
| 425 | LLVM::ModuleTranslation &moduleTranslation, | |||
| 426 | SmallVectorImpl<llvm::Value *> *continuationBlockArgs = nullptr) { | |||
| 427 | if (region.empty()) | |||
| 428 | return success(); | |||
| 429 | ||||
| 430 | // Special case for single-block regions that don't create additional blocks: | |||
| 431 | // insert operations without creating additional blocks. | |||
| 432 | if (llvm::hasSingleElement(region)) { | |||
| 433 | moduleTranslation.mapBlock(®ion.front(), builder.GetInsertBlock()); | |||
| 434 | if (failed(moduleTranslation.convertBlock( | |||
| 435 | region.front(), /*ignoreArguments=*/true, builder))) | |||
| 436 | return failure(); | |||
| 437 | ||||
| 438 | // The continuation arguments are simply the translated terminator operands. | |||
| 439 | if (continuationBlockArgs) | |||
| 440 | llvm::append_range( | |||
| 441 | *continuationBlockArgs, | |||
| 442 | moduleTranslation.lookupValues(region.front().back().getOperands())); | |||
| 443 | ||||
| 444 | // Drop the mapping that is no longer necessary so that the same region can | |||
| 445 | // be processed multiple times. | |||
| 446 | moduleTranslation.forgetMapping(region); | |||
| 447 | return success(); | |||
| 448 | } | |||
| 449 | ||||
| 450 | LogicalResult bodyGenStatus = success(); | |||
| 451 | SmallVector<llvm::PHINode *> phis; | |||
| 452 | llvm::BasicBlock *continuationBlock = convertOmpOpRegions( | |||
| 453 | region, blockName, builder, moduleTranslation, bodyGenStatus, &phis); | |||
| 454 | if (failed(bodyGenStatus)) | |||
| 455 | return failure(); | |||
| 456 | if (continuationBlockArgs) | |||
| 457 | llvm::append_range(*continuationBlockArgs, phis); | |||
| 458 | builder.SetInsertPoint(continuationBlock, | |||
| 459 | continuationBlock->getFirstInsertionPt()); | |||
| 460 | return success(); | |||
| 461 | } | |||
| 462 | ||||
| 463 | namespace { | |||
| 464 | /// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to | |||
| 465 | /// store lambdas with capture. | |||
| 466 | using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy( | |||
| 467 | llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *, | |||
| 468 | llvm::Value *&)>; | |||
| 469 | using OwningAtomicReductionGen = | |||
| 470 | std::function<llvm::OpenMPIRBuilder::InsertPointTy( | |||
| 471 | llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *, | |||
| 472 | llvm::Value *)>; | |||
| 473 | } // namespace | |||
| 474 | ||||
| 475 | /// Create an OpenMPIRBuilder-compatible reduction generator for the given | |||
| 476 | /// reduction declaration. The generator uses `builder` but ignores its | |||
| 477 | /// insertion point. | |||
| 478 | static OwningReductionGen | |||
| 479 | makeReductionGen(omp::ReductionDeclareOp decl, llvm::IRBuilderBase &builder, | |||
| 480 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 481 | // The lambda is mutable because we need access to non-const methods of decl | |||
| 482 | // (which aren't actually mutating it), and we must capture decl by-value to | |||
| 483 | // avoid the dangling reference after the parent function returns. | |||
| 484 | OwningReductionGen gen = | |||
| 485 | [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, | |||
| 486 | llvm::Value *lhs, llvm::Value *rhs, | |||
| 487 | llvm::Value *&result) mutable { | |||
| 488 | Region &reductionRegion = decl.getReductionRegion(); | |||
| 489 | moduleTranslation.mapValue(reductionRegion.front().getArgument(0), lhs); | |||
| 490 | moduleTranslation.mapValue(reductionRegion.front().getArgument(1), rhs); | |||
| 491 | builder.restoreIP(insertPoint); | |||
| 492 | SmallVector<llvm::Value *> phis; | |||
| 493 | if (failed(inlineConvertOmpRegions(reductionRegion, | |||
| 494 | "omp.reduction.nonatomic.body", | |||
| 495 | builder, moduleTranslation, &phis))) | |||
| 496 | return llvm::OpenMPIRBuilder::InsertPointTy(); | |||
| 497 | assert(phis.size() == 1)(static_cast <bool> (phis.size() == 1) ? void (0) : __assert_fail ("phis.size() == 1", "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 497, __extension__ __PRETTY_FUNCTION__)); | |||
| 498 | result = phis[0]; | |||
| 499 | return builder.saveIP(); | |||
| 500 | }; | |||
| 501 | return gen; | |||
| 502 | } | |||
| 503 | ||||
| 504 | /// Create an OpenMPIRBuilder-compatible atomic reduction generator for the | |||
| 505 | /// given reduction declaration. The generator uses `builder` but ignores its | |||
| 506 | /// insertion point. Returns null if there is no atomic region available in the | |||
| 507 | /// reduction declaration. | |||
| 508 | static OwningAtomicReductionGen | |||
| 509 | makeAtomicReductionGen(omp::ReductionDeclareOp decl, | |||
| 510 | llvm::IRBuilderBase &builder, | |||
| 511 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 512 | if (decl.getAtomicReductionRegion().empty()) | |||
| 513 | return OwningAtomicReductionGen(); | |||
| 514 | ||||
| 515 | // The lambda is mutable because we need access to non-const methods of decl | |||
| 516 | // (which aren't actually mutating it), and we must capture decl by-value to | |||
| 517 | // avoid the dangling reference after the parent function returns. | |||
| 518 | OwningAtomicReductionGen atomicGen = | |||
| 519 | [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *, | |||
| 520 | llvm::Value *lhs, llvm::Value *rhs) mutable { | |||
| 521 | Region &atomicRegion = decl.getAtomicReductionRegion(); | |||
| 522 | moduleTranslation.mapValue(atomicRegion.front().getArgument(0), lhs); | |||
| 523 | moduleTranslation.mapValue(atomicRegion.front().getArgument(1), rhs); | |||
| 524 | builder.restoreIP(insertPoint); | |||
| 525 | SmallVector<llvm::Value *> phis; | |||
| 526 | if (failed(inlineConvertOmpRegions(atomicRegion, | |||
| 527 | "omp.reduction.atomic.body", builder, | |||
| 528 | moduleTranslation, &phis))) | |||
| 529 | return llvm::OpenMPIRBuilder::InsertPointTy(); | |||
| 530 | assert(phis.empty())(static_cast <bool> (phis.empty()) ? void (0) : __assert_fail ("phis.empty()", "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 530, __extension__ __PRETTY_FUNCTION__)); | |||
| 531 | return builder.saveIP(); | |||
| 532 | }; | |||
| 533 | return atomicGen; | |||
| 534 | } | |||
| 535 | ||||
| 536 | /// Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder. | |||
| 537 | static LogicalResult | |||
| 538 | convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 539 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 540 | auto orderedOp = cast<omp::OrderedOp>(opInst); | |||
| 541 | ||||
| 542 | omp::ClauseDepend dependType = *orderedOp.getDependTypeVal(); | |||
| 543 | bool isDependSource = dependType == omp::ClauseDepend::dependsource; | |||
| 544 | unsigned numLoops = *orderedOp.getNumLoopsVal(); | |||
| 545 | SmallVector<llvm::Value *> vecValues = | |||
| 546 | moduleTranslation.lookupValues(orderedOp.getDependVecVars()); | |||
| 547 | ||||
| 548 | size_t indexVecValues = 0; | |||
| 549 | while (indexVecValues < vecValues.size()) { | |||
| 550 | SmallVector<llvm::Value *> storeValues; | |||
| 551 | storeValues.reserve(numLoops); | |||
| 552 | for (unsigned i = 0; i < numLoops; i++) { | |||
| 553 | storeValues.push_back(vecValues[indexVecValues]); | |||
| 554 | indexVecValues++; | |||
| 555 | } | |||
| 556 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = | |||
| 557 | findAllocaInsertPoint(builder, moduleTranslation); | |||
| 558 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 559 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createOrderedDepend( | |||
| 560 | ompLoc, allocaIP, numLoops, storeValues, ".cnt.addr", isDependSource)); | |||
| 561 | } | |||
| 562 | return success(); | |||
| 563 | } | |||
| 564 | ||||
| 565 | /// Converts an OpenMP 'ordered_region' operation into LLVM IR using | |||
| 566 | /// OpenMPIRBuilder. | |||
| 567 | static LogicalResult | |||
| 568 | convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 569 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 570 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 571 | auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst); | |||
| 572 | ||||
| 573 | // TODO: The code generation for ordered simd directive is not supported yet. | |||
| 574 | if (orderedRegionOp.getSimd()) | |||
| 575 | return failure(); | |||
| 576 | ||||
| 577 | // TODO: support error propagation in OpenMPIRBuilder and use it instead of | |||
| 578 | // relying on captured variables. | |||
| 579 | LogicalResult bodyGenStatus = success(); | |||
| 580 | ||||
| 581 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 582 | // OrderedOp has only one region associated with it. | |||
| 583 | auto ®ion = cast<omp::OrderedRegionOp>(opInst).getRegion(); | |||
| 584 | builder.restoreIP(codeGenIP); | |||
| 585 | convertOmpOpRegions(region, "omp.ordered.region", builder, | |||
| 586 | moduleTranslation, bodyGenStatus); | |||
| 587 | }; | |||
| 588 | ||||
| 589 | // TODO: Perform finalization actions for variables. This has to be | |||
| 590 | // called for variables which have destructors/finalizers. | |||
| 591 | auto finiCB = [&](InsertPointTy codeGenIP) {}; | |||
| 592 | ||||
| 593 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 594 | builder.restoreIP( | |||
| 595 | moduleTranslation.getOpenMPBuilder()->createOrderedThreadsSimd( | |||
| 596 | ompLoc, bodyGenCB, finiCB, !orderedRegionOp.getSimd())); | |||
| 597 | return bodyGenStatus; | |||
| 598 | } | |||
| 599 | ||||
| 600 | static LogicalResult | |||
| 601 | convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 602 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 603 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 604 | using StorableBodyGenCallbackTy = | |||
| 605 | llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy; | |||
| 606 | ||||
| 607 | auto sectionsOp = cast<omp::SectionsOp>(opInst); | |||
| 608 | ||||
| 609 | // TODO: Support the following clauses: private, firstprivate, lastprivate, | |||
| 610 | // reduction, allocate | |||
| 611 | if (!sectionsOp.getReductionVars().empty() || sectionsOp.getReductions() || | |||
| 612 | !sectionsOp.getAllocateVars().empty() || | |||
| 613 | !sectionsOp.getAllocatorsVars().empty()) | |||
| 614 | return emitError(sectionsOp.getLoc()) | |||
| 615 | << "reduction and allocate clauses are not supported for sections " | |||
| 616 | "construct"; | |||
| 617 | ||||
| 618 | LogicalResult bodyGenStatus = success(); | |||
| 619 | SmallVector<StorableBodyGenCallbackTy> sectionCBs; | |||
| 620 | ||||
| 621 | for (Operation &op : *sectionsOp.getRegion().begin()) { | |||
| 622 | auto sectionOp = dyn_cast<omp::SectionOp>(op); | |||
| 623 | if (!sectionOp) // omp.terminator | |||
| 624 | continue; | |||
| 625 | ||||
| 626 | Region ®ion = sectionOp.getRegion(); | |||
| 627 | auto sectionCB = [®ion, &builder, &moduleTranslation, &bodyGenStatus]( | |||
| 628 | InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 629 | builder.restoreIP(codeGenIP); | |||
| 630 | convertOmpOpRegions(region, "omp.section.region", builder, | |||
| 631 | moduleTranslation, bodyGenStatus); | |||
| 632 | }; | |||
| 633 | sectionCBs.push_back(sectionCB); | |||
| 634 | } | |||
| 635 | ||||
| 636 | // No sections within omp.sections operation - skip generation. This situation | |||
| 637 | // is only possible if there is only a terminator operation inside the | |||
| 638 | // sections operation | |||
| 639 | if (sectionCBs.empty()) | |||
| 640 | return success(); | |||
| 641 | ||||
| 642 | assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin()))(static_cast <bool> (isa<omp::SectionOp>(*sectionsOp .getRegion().op_begin())) ? void (0) : __assert_fail ("isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin())" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 642, __extension__ __PRETTY_FUNCTION__)); | |||
| 643 | ||||
| 644 | // TODO: Perform appropriate actions according to the data-sharing | |||
| 645 | // attribute (shared, private, firstprivate, ...) of variables. | |||
| 646 | // Currently defaults to shared. | |||
| 647 | auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &, | |||
| 648 | llvm::Value &vPtr, | |||
| 649 | llvm::Value *&replacementValue) -> InsertPointTy { | |||
| 650 | replacementValue = &vPtr; | |||
| 651 | return codeGenIP; | |||
| 652 | }; | |||
| 653 | ||||
| 654 | // TODO: Perform finalization actions for variables. This has to be | |||
| 655 | // called for variables which have destructors/finalizers. | |||
| 656 | auto finiCB = [&](InsertPointTy codeGenIP) {}; | |||
| 657 | ||||
| 658 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = | |||
| 659 | findAllocaInsertPoint(builder, moduleTranslation); | |||
| 660 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 661 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSections( | |||
| 662 | ompLoc, allocaIP, sectionCBs, privCB, finiCB, false, | |||
| 663 | sectionsOp.getNowait())); | |||
| 664 | return bodyGenStatus; | |||
| 665 | } | |||
| 666 | ||||
| 667 | /// Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder. | |||
| 668 | static LogicalResult | |||
| 669 | convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, | |||
| 670 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 671 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 672 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 673 | LogicalResult bodyGenStatus = success(); | |||
| 674 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) { | |||
| 675 | builder.restoreIP(codegenIP); | |||
| 676 | convertOmpOpRegions(singleOp.getRegion(), "omp.single.region", builder, | |||
| 677 | moduleTranslation, bodyGenStatus); | |||
| 678 | }; | |||
| 679 | auto finiCB = [&](InsertPointTy codeGenIP) {}; | |||
| 680 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSingle( | |||
| 681 | ompLoc, bodyCB, finiCB, singleOp.getNowait(), /*DidIt=*/nullptr)); | |||
| 682 | return bodyGenStatus; | |||
| 683 | } | |||
| 684 | ||||
| 685 | /// Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder. | |||
| 686 | static LogicalResult | |||
| 687 | convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, | |||
| 688 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 689 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 690 | LogicalResult bodyGenStatus = success(); | |||
| 691 | if (taskOp.getIfExpr() || taskOp.getFinalExpr() || taskOp.getUntiedAttr() || | |||
| 692 | taskOp.getMergeableAttr() || taskOp.getInReductions() || | |||
| 693 | taskOp.getPriority() || !taskOp.getAllocateVars().empty()) { | |||
| 694 | return taskOp.emitError("unhandled clauses for translation to LLVM IR"); | |||
| 695 | } | |||
| 696 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) { | |||
| 697 | builder.restoreIP(codegenIP); | |||
| 698 | convertOmpOpRegions(taskOp.getRegion(), "omp.task.region", builder, | |||
| 699 | moduleTranslation, bodyGenStatus); | |||
| 700 | }; | |||
| 701 | ||||
| 702 | SmallVector<llvm::OpenMPIRBuilder::DependData> dds; | |||
| 703 | if (!taskOp.getDependVars().empty() && taskOp.getDepends()) { | |||
| 704 | for (auto dep : | |||
| 705 | llvm::zip(taskOp.getDependVars(), taskOp.getDepends()->getValue())) { | |||
| 706 | llvm::omp::RTLDependenceKindTy type; | |||
| 707 | switch ( | |||
| 708 | std::get<1>(dep).cast<mlir::omp::ClauseTaskDependAttr>().getValue()) { | |||
| 709 | case mlir::omp::ClauseTaskDepend::taskdependin: | |||
| 710 | type = llvm::omp::RTLDependenceKindTy::DepIn; | |||
| 711 | break; | |||
| 712 | // The OpenMP runtime requires that the codegen for 'depend' clause for | |||
| 713 | // 'out' dependency kind must be the same as codegen for 'depend' clause | |||
| 714 | // with 'inout' dependency. | |||
| 715 | case mlir::omp::ClauseTaskDepend::taskdependout: | |||
| 716 | case mlir::omp::ClauseTaskDepend::taskdependinout: | |||
| 717 | type = llvm::omp::RTLDependenceKindTy::DepInOut; | |||
| 718 | break; | |||
| 719 | }; | |||
| 720 | llvm::Value *depVal = moduleTranslation.lookupValue(std::get<0>(dep)); | |||
| 721 | llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal); | |||
| 722 | dds.emplace_back(dd); | |||
| 723 | } | |||
| 724 | } | |||
| 725 | ||||
| 726 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = | |||
| 727 | findAllocaInsertPoint(builder, moduleTranslation); | |||
| 728 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 729 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTask( | |||
| 730 | ompLoc, allocaIP, bodyCB, !taskOp.getUntied(), /*Final*/ nullptr, | |||
| 731 | /*IfCondition*/ nullptr, dds)); | |||
| 732 | return bodyGenStatus; | |||
| 733 | } | |||
| 734 | ||||
| 735 | /// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder. | |||
| 736 | static LogicalResult | |||
| 737 | convertOmpTaskgroupOp(omp::TaskGroupOp tgOp, llvm::IRBuilderBase &builder, | |||
| 738 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 739 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 740 | LogicalResult bodyGenStatus = success(); | |||
| 741 | if (!tgOp.getTaskReductionVars().empty() || !tgOp.getAllocateVars().empty()) { | |||
| 742 | return tgOp.emitError("unhandled clauses for translation to LLVM IR"); | |||
| 743 | } | |||
| 744 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) { | |||
| 745 | builder.restoreIP(codegenIP); | |||
| 746 | convertOmpOpRegions(tgOp.getRegion(), "omp.taskgroup.region", builder, | |||
| 747 | moduleTranslation, bodyGenStatus); | |||
| 748 | }; | |||
| 749 | InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation); | |||
| 750 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 751 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTaskgroup( | |||
| 752 | ompLoc, allocaIP, bodyCB)); | |||
| 753 | return bodyGenStatus; | |||
| 754 | } | |||
| 755 | ||||
| 756 | /// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder. | |||
| 757 | static LogicalResult | |||
| 758 | convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 759 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 760 | auto loop = cast<omp::WsLoopOp>(opInst); | |||
| 761 | // TODO: this should be in the op verifier instead. | |||
| 762 | if (loop.getLowerBound().empty()) | |||
| 763 | return failure(); | |||
| 764 | ||||
| 765 | // Static is the default. | |||
| 766 | auto schedule = | |||
| 767 | loop.getScheduleVal().value_or(omp::ClauseScheduleKind::Static); | |||
| 768 | ||||
| 769 | // Find the loop configuration. | |||
| 770 | llvm::Value *step = moduleTranslation.lookupValue(loop.getStep()[0]); | |||
| 771 | llvm::Type *ivType = step->getType(); | |||
| 772 | llvm::Value *chunk = nullptr; | |||
| 773 | if (loop.getScheduleChunkVar()) { | |||
| 774 | llvm::Value *chunkVar = | |||
| 775 | moduleTranslation.lookupValue(loop.getScheduleChunkVar()); | |||
| 776 | chunk = builder.CreateSExtOrTrunc(chunkVar, ivType); | |||
| 777 | } | |||
| 778 | ||||
| 779 | SmallVector<omp::ReductionDeclareOp> reductionDecls; | |||
| 780 | collectReductionDecls(loop, reductionDecls); | |||
| 781 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = | |||
| 782 | findAllocaInsertPoint(builder, moduleTranslation); | |||
| 783 | ||||
| 784 | // Allocate space for privatized reduction variables. | |||
| 785 | SmallVector<llvm::Value *> privateReductionVariables; | |||
| 786 | DenseMap<Value, llvm::Value *> reductionVariableMap; | |||
| 787 | unsigned numReductions = loop.getNumReductionVars(); | |||
| 788 | privateReductionVariables.reserve(numReductions); | |||
| 789 | if (numReductions != 0) { | |||
| 790 | llvm::IRBuilderBase::InsertPointGuard guard(builder); | |||
| 791 | builder.restoreIP(allocaIP); | |||
| 792 | for (unsigned i = 0; i < numReductions; ++i) { | |||
| 793 | llvm::Value *var = builder.CreateAlloca( | |||
| 794 | moduleTranslation.convertType(reductionDecls[i].getType())); | |||
| 795 | privateReductionVariables.push_back(var); | |||
| 796 | reductionVariableMap.try_emplace(loop.getReductionVars()[i], var); | |||
| 797 | } | |||
| 798 | } | |||
| 799 | ||||
| 800 | // Store the mapping between reduction variables and their private copies on | |||
| 801 | // ModuleTranslation stack. It can be then recovered when translating | |||
| 802 | // omp.reduce operations in a separate call. | |||
| 803 | LLVM::ModuleTranslation::SaveStack<OpenMPVarMappingStackFrame> mappingGuard( | |||
| 804 | moduleTranslation, reductionVariableMap); | |||
| 805 | ||||
| 806 | // Before the loop, store the initial values of reductions into reduction | |||
| 807 | // variables. Although this could be done after allocas, we don't want to mess | |||
| 808 | // up with the alloca insertion point. | |||
| 809 | for (unsigned i = 0; i < numReductions; ++i) { | |||
| 810 | SmallVector<llvm::Value *> phis; | |||
| 811 | if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(), | |||
| 812 | "omp.reduction.neutral", builder, | |||
| 813 | moduleTranslation, &phis))) | |||
| 814 | return failure(); | |||
| 815 | assert(phis.size() == 1 && "expected one value to be yielded from the "(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from the " "reduction neutral element declaration region") ? void (0) : __assert_fail ("phis.size() == 1 && \"expected one value to be yielded from the \" \"reduction neutral element declaration region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 816, __extension__ __PRETTY_FUNCTION__)) | |||
| 816 | "reduction neutral element declaration region")(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from the " "reduction neutral element declaration region") ? void (0) : __assert_fail ("phis.size() == 1 && \"expected one value to be yielded from the \" \"reduction neutral element declaration region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 816, __extension__ __PRETTY_FUNCTION__)); | |||
| 817 | builder.CreateStore(phis[0], privateReductionVariables[i]); | |||
| 818 | } | |||
| 819 | ||||
| 820 | // Set up the source location value for OpenMP runtime. | |||
| 821 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 822 | ||||
| 823 | // Generator of the canonical loop body. | |||
| 824 | // TODO: support error propagation in OpenMPIRBuilder and use it instead of | |||
| 825 | // relying on captured variables. | |||
| 826 | SmallVector<llvm::CanonicalLoopInfo *> loopInfos; | |||
| 827 | SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints; | |||
| 828 | LogicalResult bodyGenStatus = success(); | |||
| 829 | auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) { | |||
| 830 | // Make sure further conversions know about the induction variable. | |||
| 831 | moduleTranslation.mapValue( | |||
| 832 | loop.getRegion().front().getArgument(loopInfos.size()), iv); | |||
| 833 | ||||
| 834 | // Capture the body insertion point for use in nested loops. BodyIP of the | |||
| 835 | // CanonicalLoopInfo always points to the beginning of the entry block of | |||
| 836 | // the body. | |||
| 837 | bodyInsertPoints.push_back(ip); | |||
| 838 | ||||
| 839 | if (loopInfos.size() != loop.getNumLoops() - 1) | |||
| 840 | return; | |||
| 841 | ||||
| 842 | // Convert the body of the loop. | |||
| 843 | builder.restoreIP(ip); | |||
| 844 | convertOmpOpRegions(loop.getRegion(), "omp.wsloop.region", builder, | |||
| 845 | moduleTranslation, bodyGenStatus); | |||
| 846 | }; | |||
| 847 | ||||
| 848 | // Delegate actual loop construction to the OpenMP IRBuilder. | |||
| 849 | // TODO: this currently assumes WsLoop is semantically similar to SCF loop, | |||
| 850 | // i.e. it has a positive step, uses signed integer semantics. Reconsider | |||
| 851 | // this code when WsLoop clearly supports more cases. | |||
| 852 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 853 | for (unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) { | |||
| 854 | llvm::Value *lowerBound = | |||
| 855 | moduleTranslation.lookupValue(loop.getLowerBound()[i]); | |||
| 856 | llvm::Value *upperBound = | |||
| 857 | moduleTranslation.lookupValue(loop.getUpperBound()[i]); | |||
| 858 | llvm::Value *step = moduleTranslation.lookupValue(loop.getStep()[i]); | |||
| 859 | ||||
| 860 | // Make sure loop trip count are emitted in the preheader of the outermost | |||
| 861 | // loop at the latest so that they are all available for the new collapsed | |||
| 862 | // loop will be created below. | |||
| 863 | llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc; | |||
| 864 | llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP; | |||
| 865 | if (i != 0) { | |||
| 866 | loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back()); | |||
| 867 | computeIP = loopInfos.front()->getPreheaderIP(); | |||
| 868 | } | |||
| 869 | loopInfos.push_back(ompBuilder->createCanonicalLoop( | |||
| 870 | loc, bodyGen, lowerBound, upperBound, step, | |||
| 871 | /*IsSigned=*/true, loop.getInclusive(), computeIP)); | |||
| 872 | ||||
| 873 | if (failed(bodyGenStatus)) | |||
| 874 | return failure(); | |||
| 875 | } | |||
| 876 | ||||
| 877 | // Collapse loops. Store the insertion point because LoopInfos may get | |||
| 878 | // invalidated. | |||
| 879 | llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP(); | |||
| 880 | llvm::CanonicalLoopInfo *loopInfo = | |||
| 881 | ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {}); | |||
| 882 | ||||
| 883 | allocaIP = findAllocaInsertPoint(builder, moduleTranslation); | |||
| 884 | ||||
| 885 | // TODO: Handle doacross loops when the ordered clause has a parameter. | |||
| 886 | bool isOrdered = loop.getOrderedVal().has_value(); | |||
| 887 | std::optional<omp::ScheduleModifier> scheduleModifier = | |||
| 888 | loop.getScheduleModifier(); | |||
| 889 | bool isSimd = loop.getSimdModifier(); | |||
| 890 | ||||
| 891 | ompBuilder->applyWorkshareLoop( | |||
| 892 | ompLoc.DL, loopInfo, allocaIP, !loop.getNowait(), | |||
| 893 | convertToScheduleKind(schedule), chunk, isSimd, | |||
| 894 | scheduleModifier == omp::ScheduleModifier::monotonic, | |||
| 895 | scheduleModifier == omp::ScheduleModifier::nonmonotonic, isOrdered); | |||
| 896 | ||||
| 897 | // Continue building IR after the loop. Note that the LoopInfo returned by | |||
| 898 | // `collapseLoops` points inside the outermost loop and is intended for | |||
| 899 | // potential further loop transformations. Use the insertion point stored | |||
| 900 | // before collapsing loops instead. | |||
| 901 | builder.restoreIP(afterIP); | |||
| 902 | ||||
| 903 | // Process the reductions if required. | |||
| 904 | if (numReductions == 0) | |||
| 905 | return success(); | |||
| 906 | ||||
| 907 | // Create the reduction generators. We need to own them here because | |||
| 908 | // ReductionInfo only accepts references to the generators. | |||
| 909 | SmallVector<OwningReductionGen> owningReductionGens; | |||
| 910 | SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens; | |||
| 911 | for (unsigned i = 0; i < numReductions; ++i) { | |||
| 912 | owningReductionGens.push_back( | |||
| 913 | makeReductionGen(reductionDecls[i], builder, moduleTranslation)); | |||
| 914 | owningAtomicReductionGens.push_back( | |||
| 915 | makeAtomicReductionGen(reductionDecls[i], builder, moduleTranslation)); | |||
| 916 | } | |||
| 917 | ||||
| 918 | // Collect the reduction information. | |||
| 919 | SmallVector<llvm::OpenMPIRBuilder::ReductionInfo> reductionInfos; | |||
| 920 | reductionInfos.reserve(numReductions); | |||
| 921 | for (unsigned i = 0; i < numReductions; ++i) { | |||
| 922 | llvm::OpenMPIRBuilder::AtomicReductionGenTy atomicGen = nullptr; | |||
| 923 | if (owningAtomicReductionGens[i]) | |||
| 924 | atomicGen = owningAtomicReductionGens[i]; | |||
| 925 | llvm::Value *variable = | |||
| 926 | moduleTranslation.lookupValue(loop.getReductionVars()[i]); | |||
| 927 | reductionInfos.push_back( | |||
| 928 | {moduleTranslation.convertType(reductionDecls[i].getType()), variable, | |||
| 929 | privateReductionVariables[i], owningReductionGens[i], atomicGen}); | |||
| 930 | } | |||
| 931 | ||||
| 932 | // The call to createReductions below expects the block to have a | |||
| 933 | // terminator. Create an unreachable instruction to serve as terminator | |||
| 934 | // and remove it later. | |||
| 935 | llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable(); | |||
| 936 | builder.SetInsertPoint(tempTerminator); | |||
| 937 | llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint = | |||
| 938 | ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos, | |||
| 939 | loop.getNowait()); | |||
| 940 | if (!contInsertPoint.getBlock()) | |||
| 941 | return loop->emitOpError() << "failed to convert reductions"; | |||
| 942 | auto nextInsertionPoint = | |||
| 943 | ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for); | |||
| 944 | tempTerminator->eraseFromParent(); | |||
| 945 | builder.restoreIP(nextInsertionPoint); | |||
| 946 | ||||
| 947 | return success(); | |||
| 948 | } | |||
| 949 | ||||
| 950 | /// Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder. | |||
| 951 | static LogicalResult | |||
| 952 | convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 953 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 954 | auto loop = cast<omp::SimdLoopOp>(opInst); | |||
| 955 | ||||
| 956 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 957 | ||||
| 958 | // Generator of the canonical loop body. | |||
| 959 | // TODO: support error propagation in OpenMPIRBuilder and use it instead of | |||
| 960 | // relying on captured variables. | |||
| 961 | SmallVector<llvm::CanonicalLoopInfo *> loopInfos; | |||
| 962 | SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints; | |||
| 963 | LogicalResult bodyGenStatus = success(); | |||
| 964 | auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) { | |||
| 965 | // Make sure further conversions know about the induction variable. | |||
| 966 | moduleTranslation.mapValue( | |||
| 967 | loop.getRegion().front().getArgument(loopInfos.size()), iv); | |||
| 968 | ||||
| 969 | // Capture the body insertion point for use in nested loops. BodyIP of the | |||
| 970 | // CanonicalLoopInfo always points to the beginning of the entry block of | |||
| 971 | // the body. | |||
| 972 | bodyInsertPoints.push_back(ip); | |||
| 973 | ||||
| 974 | if (loopInfos.size() != loop.getNumLoops() - 1) | |||
| 975 | return; | |||
| 976 | ||||
| 977 | // Convert the body of the loop. | |||
| 978 | builder.restoreIP(ip); | |||
| 979 | convertOmpOpRegions(loop.getRegion(), "omp.simdloop.region", builder, | |||
| 980 | moduleTranslation, bodyGenStatus); | |||
| 981 | }; | |||
| 982 | ||||
| 983 | // Delegate actual loop construction to the OpenMP IRBuilder. | |||
| 984 | // TODO: this currently assumes SimdLoop is semantically similar to SCF loop, | |||
| 985 | // i.e. it has a positive step, uses signed integer semantics. Reconsider | |||
| 986 | // this code when SimdLoop clearly supports more cases. | |||
| 987 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 988 | for (unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) { | |||
| 989 | llvm::Value *lowerBound = | |||
| 990 | moduleTranslation.lookupValue(loop.getLowerBound()[i]); | |||
| 991 | llvm::Value *upperBound = | |||
| 992 | moduleTranslation.lookupValue(loop.getUpperBound()[i]); | |||
| 993 | llvm::Value *step = moduleTranslation.lookupValue(loop.getStep()[i]); | |||
| 994 | ||||
| 995 | // Make sure loop trip count are emitted in the preheader of the outermost | |||
| 996 | // loop at the latest so that they are all available for the new collapsed | |||
| 997 | // loop will be created below. | |||
| 998 | llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc; | |||
| 999 | llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP; | |||
| 1000 | if (i != 0) { | |||
| 1001 | loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(), | |||
| 1002 | ompLoc.DL); | |||
| 1003 | computeIP = loopInfos.front()->getPreheaderIP(); | |||
| 1004 | } | |||
| 1005 | loopInfos.push_back(ompBuilder->createCanonicalLoop( | |||
| 1006 | loc, bodyGen, lowerBound, upperBound, step, | |||
| 1007 | /*IsSigned=*/true, /*Inclusive=*/true, computeIP)); | |||
| 1008 | ||||
| 1009 | if (failed(bodyGenStatus)) | |||
| 1010 | return failure(); | |||
| 1011 | } | |||
| 1012 | ||||
| 1013 | // Collapse loops. | |||
| 1014 | llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP(); | |||
| 1015 | llvm::CanonicalLoopInfo *loopInfo = | |||
| 1016 | ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {}); | |||
| 1017 | ||||
| 1018 | llvm::ConstantInt *simdlen = nullptr; | |||
| 1019 | if (std::optional<uint64_t> simdlenVar = loop.getSimdlen()) | |||
| 1020 | simdlen = builder.getInt64(simdlenVar.value()); | |||
| 1021 | ||||
| 1022 | llvm::ConstantInt *safelen = nullptr; | |||
| 1023 | if (std::optional<uint64_t> safelenVar = loop.getSafelen()) | |||
| 1024 | safelen = builder.getInt64(safelenVar.value()); | |||
| 1025 | ||||
| 1026 | llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars; | |||
| 1027 | ompBuilder->applySimd( | |||
| 1028 | loopInfo, alignedVars, | |||
| 1029 | loop.getIfExpr() ? moduleTranslation.lookupValue(loop.getIfExpr()) | |||
| 1030 | : nullptr, | |||
| 1031 | llvm::omp::OrderKind::OMP_ORDER_unknown, simdlen, safelen); | |||
| 1032 | ||||
| 1033 | builder.restoreIP(afterIP); | |||
| 1034 | return success(); | |||
| 1035 | } | |||
| 1036 | ||||
| 1037 | /// Convert an Atomic Ordering attribute to llvm::AtomicOrdering. | |||
| 1038 | llvm::AtomicOrdering | |||
| 1039 | convertAtomicOrdering(std::optional<omp::ClauseMemoryOrderKind> ao) { | |||
| 1040 | if (!ao) | |||
| 1041 | return llvm::AtomicOrdering::Monotonic; // Default Memory Ordering | |||
| 1042 | ||||
| 1043 | switch (*ao) { | |||
| 1044 | case omp::ClauseMemoryOrderKind::Seq_cst: | |||
| 1045 | return llvm::AtomicOrdering::SequentiallyConsistent; | |||
| 1046 | case omp::ClauseMemoryOrderKind::Acq_rel: | |||
| 1047 | return llvm::AtomicOrdering::AcquireRelease; | |||
| 1048 | case omp::ClauseMemoryOrderKind::Acquire: | |||
| 1049 | return llvm::AtomicOrdering::Acquire; | |||
| 1050 | case omp::ClauseMemoryOrderKind::Release: | |||
| 1051 | return llvm::AtomicOrdering::Release; | |||
| 1052 | case omp::ClauseMemoryOrderKind::Relaxed: | |||
| 1053 | return llvm::AtomicOrdering::Monotonic; | |||
| 1054 | } | |||
| 1055 | llvm_unreachable("Unknown ClauseMemoryOrderKind kind")::llvm::llvm_unreachable_internal("Unknown ClauseMemoryOrderKind kind" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1055); | |||
| 1056 | } | |||
| 1057 | ||||
| 1058 | /// Convert omp.atomic.read operation to LLVM IR. | |||
| 1059 | static LogicalResult | |||
| 1060 | convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 1061 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1062 | ||||
| 1063 | auto readOp = cast<omp::AtomicReadOp>(opInst); | |||
| 1064 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1065 | ||||
| 1066 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1067 | ||||
| 1068 | llvm::AtomicOrdering AO = convertAtomicOrdering(readOp.getMemoryOrderVal()); | |||
| 1069 | llvm::Value *x = moduleTranslation.lookupValue(readOp.getX()); | |||
| 1070 | llvm::Value *v = moduleTranslation.lookupValue(readOp.getV()); | |||
| 1071 | ||||
| 1072 | llvm::Type *elementType = | |||
| 1073 | moduleTranslation.convertType(readOp.getElementType()); | |||
| 1074 | ||||
| 1075 | llvm::OpenMPIRBuilder::AtomicOpValue V = {v, elementType, false, false}; | |||
| 1076 | llvm::OpenMPIRBuilder::AtomicOpValue X = {x, elementType, false, false}; | |||
| 1077 | builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO)); | |||
| 1078 | return success(); | |||
| 1079 | } | |||
| 1080 | ||||
| 1081 | /// Converts an omp.atomic.write operation to LLVM IR. | |||
| 1082 | static LogicalResult | |||
| 1083 | convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 1084 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1085 | auto writeOp = cast<omp::AtomicWriteOp>(opInst); | |||
| 1086 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1087 | ||||
| 1088 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1089 | llvm::AtomicOrdering ao = convertAtomicOrdering(writeOp.getMemoryOrderVal()); | |||
| 1090 | llvm::Value *expr = moduleTranslation.lookupValue(writeOp.getValue()); | |||
| 1091 | llvm::Value *dest = moduleTranslation.lookupValue(writeOp.getAddress()); | |||
| 1092 | llvm::Type *ty = moduleTranslation.convertType(writeOp.getValue().getType()); | |||
| 1093 | llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty, /*isSigned=*/false, | |||
| 1094 | /*isVolatile=*/false}; | |||
| 1095 | builder.restoreIP(ompBuilder->createAtomicWrite(ompLoc, x, expr, ao)); | |||
| 1096 | return success(); | |||
| 1097 | } | |||
| 1098 | ||||
| 1099 | /// Converts an LLVM dialect binary operation to the corresponding enum value | |||
| 1100 | /// for `atomicrmw` supported binary operation. | |||
| 1101 | llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) { | |||
| 1102 | return llvm::TypeSwitch<Operation *, llvm::AtomicRMWInst::BinOp>(&op) | |||
| 1103 | .Case([&](LLVM::AddOp) { return llvm::AtomicRMWInst::BinOp::Add; }) | |||
| 1104 | .Case([&](LLVM::SubOp) { return llvm::AtomicRMWInst::BinOp::Sub; }) | |||
| 1105 | .Case([&](LLVM::AndOp) { return llvm::AtomicRMWInst::BinOp::And; }) | |||
| 1106 | .Case([&](LLVM::OrOp) { return llvm::AtomicRMWInst::BinOp::Or; }) | |||
| 1107 | .Case([&](LLVM::XOrOp) { return llvm::AtomicRMWInst::BinOp::Xor; }) | |||
| 1108 | .Case([&](LLVM::UMaxOp) { return llvm::AtomicRMWInst::BinOp::UMax; }) | |||
| 1109 | .Case([&](LLVM::UMinOp) { return llvm::AtomicRMWInst::BinOp::UMin; }) | |||
| 1110 | .Case([&](LLVM::FAddOp) { return llvm::AtomicRMWInst::BinOp::FAdd; }) | |||
| 1111 | .Case([&](LLVM::FSubOp) { return llvm::AtomicRMWInst::BinOp::FSub; }) | |||
| 1112 | .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP); | |||
| 1113 | } | |||
| 1114 | ||||
| 1115 | /// Converts an OpenMP atomic update operation using OpenMPIRBuilder. | |||
| 1116 | static LogicalResult | |||
| 1117 | convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, | |||
| 1118 | llvm::IRBuilderBase &builder, | |||
| 1119 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1120 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1121 | ||||
| 1122 | // Convert values and types. | |||
| 1123 | auto &innerOpList = opInst.getRegion().front().getOperations(); | |||
| 1124 | if (innerOpList.size() != 2) | |||
| 1125 | return opInst.emitError("exactly two operations are allowed inside an " | |||
| 1126 | "atomic update region while lowering to LLVM IR"); | |||
| 1127 | ||||
| 1128 | Operation &innerUpdateOp = innerOpList.front(); | |||
| 1129 | ||||
| 1130 | if (innerUpdateOp.getNumOperands() != 2 || | |||
| 1131 | !llvm::is_contained(innerUpdateOp.getOperands(), | |||
| 1132 | opInst.getRegion().getArgument(0))) | |||
| 1133 | return opInst.emitError( | |||
| 1134 | "the update operation inside the region must be a binary operation and " | |||
| 1135 | "that update operation must have the region argument as an operand"); | |||
| 1136 | ||||
| 1137 | llvm::AtomicRMWInst::BinOp binop = convertBinOpToAtomic(innerUpdateOp); | |||
| 1138 | ||||
| 1139 | bool isXBinopExpr = | |||
| 1140 | innerUpdateOp.getNumOperands() > 0 && | |||
| 1141 | innerUpdateOp.getOperand(0) == opInst.getRegion().getArgument(0); | |||
| 1142 | ||||
| 1143 | mlir::Value mlirExpr = (isXBinopExpr ? innerUpdateOp.getOperand(1) | |||
| 1144 | : innerUpdateOp.getOperand(0)); | |||
| 1145 | llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr); | |||
| 1146 | llvm::Value *llvmX = moduleTranslation.lookupValue(opInst.getX()); | |||
| 1147 | llvm::Type *llvmXElementType = moduleTranslation.convertType( | |||
| 1148 | opInst.getRegion().getArgument(0).getType()); | |||
| 1149 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType, | |||
| 1150 | /*isSigned=*/false, | |||
| 1151 | /*isVolatile=*/false}; | |||
| 1152 | ||||
| 1153 | llvm::AtomicOrdering atomicOrdering = | |||
| 1154 | convertAtomicOrdering(opInst.getMemoryOrderVal()); | |||
| 1155 | ||||
| 1156 | // Generate update code. | |||
| 1157 | LogicalResult updateGenStatus = success(); | |||
| 1158 | auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus]( | |||
| 1159 | llvm::Value *atomicx, | |||
| 1160 | llvm::IRBuilder<> &builder) -> llvm::Value * { | |||
| 1161 | Block &bb = *opInst.getRegion().begin(); | |||
| 1162 | moduleTranslation.mapValue(*opInst.getRegion().args_begin(), atomicx); | |||
| 1163 | moduleTranslation.mapBlock(&bb, builder.GetInsertBlock()); | |||
| 1164 | if (failed(moduleTranslation.convertBlock(bb, true, builder))) { | |||
| 1165 | updateGenStatus = (opInst.emitError() | |||
| 1166 | << "unable to convert update operation to llvm IR"); | |||
| 1167 | return nullptr; | |||
| 1168 | } | |||
| 1169 | omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator()); | |||
| 1170 | assert(yieldop && yieldop.getResults().size() == 1 &&(static_cast <bool> (yieldop && yieldop.getResults ().size() == 1 && "terminator must be omp.yield op and it must have exactly one " "argument") ? void (0) : __assert_fail ("yieldop && yieldop.getResults().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1172, __extension__ __PRETTY_FUNCTION__)) | |||
| 1171 | "terminator must be omp.yield op and it must have exactly one "(static_cast <bool> (yieldop && yieldop.getResults ().size() == 1 && "terminator must be omp.yield op and it must have exactly one " "argument") ? void (0) : __assert_fail ("yieldop && yieldop.getResults().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1172, __extension__ __PRETTY_FUNCTION__)) | |||
| 1172 | "argument")(static_cast <bool> (yieldop && yieldop.getResults ().size() == 1 && "terminator must be omp.yield op and it must have exactly one " "argument") ? void (0) : __assert_fail ("yieldop && yieldop.getResults().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1172, __extension__ __PRETTY_FUNCTION__)); | |||
| 1173 | return moduleTranslation.lookupValue(yieldop.getResults()[0]); | |||
| 1174 | }; | |||
| 1175 | ||||
| 1176 | // Handle ambiguous alloca, if any. | |||
| 1177 | auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation); | |||
| 1178 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1179 | builder.restoreIP(ompBuilder->createAtomicUpdate( | |||
| 1180 | ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn, | |||
| 1181 | isXBinopExpr)); | |||
| 1182 | return updateGenStatus; | |||
| 1183 | } | |||
| 1184 | ||||
| 1185 | static LogicalResult | |||
| 1186 | convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, | |||
| 1187 | llvm::IRBuilderBase &builder, | |||
| 1188 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1189 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1190 | mlir::Value mlirExpr; | |||
| 1191 | bool isXBinopExpr = false, isPostfixUpdate = false; | |||
| 1192 | llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP; | |||
| 1193 | ||||
| 1194 | omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp(); | |||
| 1195 | omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp(); | |||
| 1196 | ||||
| 1197 | assert((atomicUpdateOp || atomicWriteOp) &&(static_cast <bool> ((atomicUpdateOp || atomicWriteOp) && "internal op must be an atomic.update or atomic.write op") ? void (0) : __assert_fail ("(atomicUpdateOp || atomicWriteOp) && \"internal op must be an atomic.update or atomic.write op\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1198, __extension__ __PRETTY_FUNCTION__)) | |||
| 1198 | "internal op must be an atomic.update or atomic.write op")(static_cast <bool> ((atomicUpdateOp || atomicWriteOp) && "internal op must be an atomic.update or atomic.write op") ? void (0) : __assert_fail ("(atomicUpdateOp || atomicWriteOp) && \"internal op must be an atomic.update or atomic.write op\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1198, __extension__ __PRETTY_FUNCTION__)); | |||
| 1199 | ||||
| 1200 | if (atomicWriteOp) { | |||
| 1201 | isPostfixUpdate = true; | |||
| 1202 | mlirExpr = atomicWriteOp.getValue(); | |||
| 1203 | } else { | |||
| 1204 | isPostfixUpdate = atomicCaptureOp.getSecondOp() == | |||
| 1205 | atomicCaptureOp.getAtomicUpdateOp().getOperation(); | |||
| 1206 | auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations(); | |||
| 1207 | if (innerOpList.size() != 2) | |||
| 1208 | return atomicUpdateOp.emitError( | |||
| 1209 | "exactly two operations are allowed inside an " | |||
| 1210 | "atomic update region while lowering to LLVM IR"); | |||
| 1211 | Operation *innerUpdateOp = atomicUpdateOp.getFirstOp(); | |||
| 1212 | if (innerUpdateOp->getNumOperands() != 2 || | |||
| 1213 | !llvm::is_contained(innerUpdateOp->getOperands(), | |||
| 1214 | atomicUpdateOp.getRegion().getArgument(0))) | |||
| 1215 | return atomicUpdateOp.emitError( | |||
| 1216 | "the update operation inside the region must be a binary operation " | |||
| 1217 | "and that update operation must have the region argument as an " | |||
| 1218 | "operand"); | |||
| 1219 | binop = convertBinOpToAtomic(*innerUpdateOp); | |||
| 1220 | ||||
| 1221 | isXBinopExpr = innerUpdateOp->getOperand(0) == | |||
| 1222 | atomicUpdateOp.getRegion().getArgument(0); | |||
| 1223 | ||||
| 1224 | mlirExpr = (isXBinopExpr ? innerUpdateOp->getOperand(1) | |||
| 1225 | : innerUpdateOp->getOperand(0)); | |||
| 1226 | } | |||
| 1227 | ||||
| 1228 | llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr); | |||
| 1229 | llvm::Value *llvmX = | |||
| 1230 | moduleTranslation.lookupValue(atomicCaptureOp.getAtomicReadOp().getX()); | |||
| 1231 | llvm::Value *llvmV = | |||
| 1232 | moduleTranslation.lookupValue(atomicCaptureOp.getAtomicReadOp().getV()); | |||
| 1233 | llvm::Type *llvmXElementType = moduleTranslation.convertType( | |||
| 1234 | atomicCaptureOp.getAtomicReadOp().getElementType()); | |||
| 1235 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType, | |||
| 1236 | /*isSigned=*/false, | |||
| 1237 | /*isVolatile=*/false}; | |||
| 1238 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType, | |||
| 1239 | /*isSigned=*/false, | |||
| 1240 | /*isVolatile=*/false}; | |||
| 1241 | ||||
| 1242 | llvm::AtomicOrdering atomicOrdering = | |||
| 1243 | convertAtomicOrdering(atomicCaptureOp.getMemoryOrderVal()); | |||
| 1244 | ||||
| 1245 | LogicalResult updateGenStatus = success(); | |||
| 1246 | auto updateFn = [&](llvm::Value *atomicx, | |||
| 1247 | llvm::IRBuilder<> &builder) -> llvm::Value * { | |||
| 1248 | if (atomicWriteOp) | |||
| 1249 | return moduleTranslation.lookupValue(atomicWriteOp.getValue()); | |||
| 1250 | Block &bb = *atomicUpdateOp.getRegion().begin(); | |||
| 1251 | moduleTranslation.mapValue(*atomicUpdateOp.getRegion().args_begin(), | |||
| 1252 | atomicx); | |||
| 1253 | moduleTranslation.mapBlock(&bb, builder.GetInsertBlock()); | |||
| 1254 | if (failed(moduleTranslation.convertBlock(bb, true, builder))) { | |||
| 1255 | updateGenStatus = (atomicUpdateOp.emitError() | |||
| 1256 | << "unable to convert update operation to llvm IR"); | |||
| 1257 | return nullptr; | |||
| 1258 | } | |||
| 1259 | omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator()); | |||
| 1260 | assert(yieldop && yieldop.getResults().size() == 1 &&(static_cast <bool> (yieldop && yieldop.getResults ().size() == 1 && "terminator must be omp.yield op and it must have exactly one " "argument") ? void (0) : __assert_fail ("yieldop && yieldop.getResults().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1262, __extension__ __PRETTY_FUNCTION__)) | |||
| 1261 | "terminator must be omp.yield op and it must have exactly one "(static_cast <bool> (yieldop && yieldop.getResults ().size() == 1 && "terminator must be omp.yield op and it must have exactly one " "argument") ? void (0) : __assert_fail ("yieldop && yieldop.getResults().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1262, __extension__ __PRETTY_FUNCTION__)) | |||
| 1262 | "argument")(static_cast <bool> (yieldop && yieldop.getResults ().size() == 1 && "terminator must be omp.yield op and it must have exactly one " "argument") ? void (0) : __assert_fail ("yieldop && yieldop.getResults().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1262, __extension__ __PRETTY_FUNCTION__)); | |||
| 1263 | return moduleTranslation.lookupValue(yieldop.getResults()[0]); | |||
| 1264 | }; | |||
| 1265 | ||||
| 1266 | // Handle ambiguous alloca, if any. | |||
| 1267 | auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation); | |||
| 1268 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1269 | builder.restoreIP(ompBuilder->createAtomicCapture( | |||
| 1270 | ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering, | |||
| 1271 | binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr)); | |||
| 1272 | return updateGenStatus; | |||
| 1273 | } | |||
| 1274 | ||||
| 1275 | /// Converts an OpenMP reduction operation using OpenMPIRBuilder. Expects the | |||
| 1276 | /// mapping between reduction variables and their private equivalents to have | |||
| 1277 | /// been stored on the ModuleTranslation stack. Currently only supports | |||
| 1278 | /// reduction within WsLoopOp, but can be easily extended. | |||
| 1279 | static LogicalResult | |||
| 1280 | convertOmpReductionOp(omp::ReductionOp reductionOp, | |||
| 1281 | llvm::IRBuilderBase &builder, | |||
| 1282 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1283 | // Find the declaration that corresponds to the reduction op. | |||
| 1284 | auto reductionContainer = reductionOp->getParentOfType<omp::WsLoopOp>(); | |||
| 1285 | omp::ReductionDeclareOp declaration = | |||
| 1286 | findReductionDecl(reductionContainer, reductionOp); | |||
| 1287 | assert(declaration && "could not find reduction declaration")(static_cast <bool> (declaration && "could not find reduction declaration" ) ? void (0) : __assert_fail ("declaration && \"could not find reduction declaration\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1287, __extension__ __PRETTY_FUNCTION__)); | |||
| 1288 | ||||
| 1289 | // Retrieve the mapping between reduction variables and their private | |||
| 1290 | // equivalents. | |||
| 1291 | const DenseMap<Value, llvm::Value *> *reductionVariableMap = nullptr; | |||
| 1292 | moduleTranslation.stackWalk<OpenMPVarMappingStackFrame>( | |||
| 1293 | [&](const OpenMPVarMappingStackFrame &frame) { | |||
| 1294 | reductionVariableMap = &frame.mapping; | |||
| 1295 | return WalkResult::interrupt(); | |||
| 1296 | }); | |||
| 1297 | assert(reductionVariableMap && "couldn't find private reduction variables")(static_cast <bool> (reductionVariableMap && "couldn't find private reduction variables" ) ? void (0) : __assert_fail ("reductionVariableMap && \"couldn't find private reduction variables\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1297, __extension__ __PRETTY_FUNCTION__)); | |||
| 1298 | ||||
| 1299 | // Translate the reduction operation by emitting the body of the corresponding | |||
| 1300 | // reduction declaration. | |||
| 1301 | Region &reductionRegion = declaration.getReductionRegion(); | |||
| 1302 | llvm::Value *privateReductionVar = | |||
| 1303 | reductionVariableMap->lookup(reductionOp.getAccumulator()); | |||
| 1304 | llvm::Value *reductionVal = builder.CreateLoad( | |||
| 1305 | moduleTranslation.convertType(reductionOp.getOperand().getType()), | |||
| 1306 | privateReductionVar); | |||
| 1307 | ||||
| 1308 | moduleTranslation.mapValue(reductionRegion.front().getArgument(0), | |||
| 1309 | reductionVal); | |||
| 1310 | moduleTranslation.mapValue( | |||
| 1311 | reductionRegion.front().getArgument(1), | |||
| 1312 | moduleTranslation.lookupValue(reductionOp.getOperand())); | |||
| 1313 | ||||
| 1314 | SmallVector<llvm::Value *> phis; | |||
| 1315 | if (failed(inlineConvertOmpRegions(reductionRegion, "omp.reduction.body", | |||
| 1316 | builder, moduleTranslation, &phis))) | |||
| 1317 | return failure(); | |||
| 1318 | assert(phis.size() == 1 && "expected one value to be yielded from "(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from " "the reduction body declaration region") ? void (0) : __assert_fail ("phis.size() == 1 && \"expected one value to be yielded from \" \"the reduction body declaration region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1319, __extension__ __PRETTY_FUNCTION__)) | |||
| 1319 | "the reduction body declaration region")(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from " "the reduction body declaration region") ? void (0) : __assert_fail ("phis.size() == 1 && \"expected one value to be yielded from \" \"the reduction body declaration region\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1319, __extension__ __PRETTY_FUNCTION__)); | |||
| 1320 | builder.CreateStore(phis[0], privateReductionVar); | |||
| 1321 | return success(); | |||
| 1322 | } | |||
| 1323 | ||||
| 1324 | /// Converts an OpenMP Threadprivate operation into LLVM IR using | |||
| 1325 | /// OpenMPIRBuilder. | |||
| 1326 | static LogicalResult | |||
| 1327 | convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 1328 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1329 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1330 | auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst); | |||
| 1331 | ||||
| 1332 | Value symAddr = threadprivateOp.getSymAddr(); | |||
| 1333 | auto *symOp = symAddr.getDefiningOp(); | |||
| 1334 | if (!isa<LLVM::AddressOfOp>(symOp)) | |||
| 1335 | return opInst.emitError("Addressing symbol not found"); | |||
| 1336 | LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp); | |||
| 1337 | ||||
| 1338 | LLVM::GlobalOp global = | |||
| 1339 | addressOfOp.getGlobal(moduleTranslation.symbolTable()); | |||
| 1340 | llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(global); | |||
| 1341 | llvm::Value *data = | |||
| 1342 | builder.CreateBitCast(globalValue, builder.getInt8PtrTy()); | |||
| 1343 | llvm::Type *type = globalValue->getValueType(); | |||
| 1344 | llvm::TypeSize typeSize = | |||
| 1345 | builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize( | |||
| 1346 | type); | |||
| 1347 | llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedValue()); | |||
| 1348 | llvm::StringRef suffix = llvm::StringRef(".cache", 6); | |||
| 1349 | std::string cacheName = (Twine(global.getSymName()).concat(suffix)).str(); | |||
| 1350 | // Emit runtime function and bitcast its type (i8*) to real data type. | |||
| 1351 | llvm::Value *callInst = | |||
| 1352 | moduleTranslation.getOpenMPBuilder()->createCachedThreadPrivate( | |||
| 1353 | ompLoc, data, size, cacheName); | |||
| 1354 | llvm::Value *result = builder.CreateBitCast(callInst, globalValue->getType()); | |||
| 1355 | moduleTranslation.mapValue(opInst.getResult(0), result); | |||
| 1356 | return success(); | |||
| 1357 | } | |||
| 1358 | ||||
| 1359 | /// Process MapOperands for Target Data directives. | |||
| 1360 | static LogicalResult processMapOperand( | |||
| 1361 | llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, | |||
| 1362 | const SmallVector<Value> &mapOperands, const ArrayAttr &mapTypes, | |||
| 1363 | SmallVector<uint64_t> &mapTypeFlags, | |||
| 1364 | SmallVectorImpl<llvm::Constant *> &mapNames, | |||
| 1365 | struct llvm::OpenMPIRBuilder::MapperAllocas &mapperAllocas) { | |||
| 1366 | auto numMapOperands = mapOperands.size(); | |||
| 1367 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1368 | llvm::PointerType *i8PtrTy = builder.getInt8PtrTy(); | |||
| 1369 | llvm::ArrayType *arrI8PtrTy = llvm::ArrayType::get(i8PtrTy, numMapOperands); | |||
| 1370 | llvm::IntegerType *i64Ty = builder.getInt64Ty(); | |||
| 1371 | llvm::ArrayType *arrI64Ty = llvm::ArrayType::get(i64Ty, numMapOperands); | |||
| 1372 | ||||
| 1373 | unsigned index = 0; | |||
| 1374 | for (const auto &mapOp : mapOperands) { | |||
| 1375 | const auto &mapTypeOp = mapTypes[index]; | |||
| 1376 | ||||
| 1377 | llvm::Value *mapOpValue = moduleTranslation.lookupValue(mapOp); | |||
| 1378 | llvm::Value *mapOpPtrBase; | |||
| 1379 | llvm::Value *mapOpPtr; | |||
| 1380 | llvm::Value *mapOpSize; | |||
| 1381 | ||||
| 1382 | if (mapOp.getType().isa<LLVM::LLVMPointerType>()) { | |||
| 1383 | mapOpPtrBase = mapOpValue; | |||
| 1384 | mapOpPtr = mapOpValue; | |||
| 1385 | mapOpSize = ompBuilder->getSizeInBytes(mapOpValue); | |||
| 1386 | } else { | |||
| 1387 | return failure(); | |||
| 1388 | } | |||
| 1389 | ||||
| 1390 | // Store base pointer extracted from operand into the i-th position of | |||
| 1391 | // argBase. | |||
| 1392 | llvm::Value *ptrBaseGEP = builder.CreateInBoundsGEP( | |||
| 1393 | arrI8PtrTy, mapperAllocas.ArgsBase, | |||
| 1394 | {builder.getInt32(0), builder.getInt32(index)}); | |||
| 1395 | llvm::Value *ptrBaseCast = builder.CreateBitCast( | |||
| 1396 | ptrBaseGEP, mapOpPtrBase->getType()->getPointerTo()); | |||
| 1397 | builder.CreateStore(mapOpPtrBase, ptrBaseCast); | |||
| 1398 | ||||
| 1399 | // Store pointer extracted from operand into the i-th position of args. | |||
| 1400 | llvm::Value *ptrGEP = builder.CreateInBoundsGEP( | |||
| 1401 | arrI8PtrTy, mapperAllocas.Args, | |||
| 1402 | {builder.getInt32(0), builder.getInt32(index)}); | |||
| 1403 | llvm::Value *ptrCast = | |||
| 1404 | builder.CreateBitCast(ptrGEP, mapOpPtr->getType()->getPointerTo()); | |||
| 1405 | builder.CreateStore(mapOpPtr, ptrCast); | |||
| 1406 | ||||
| 1407 | // Store size extracted from operand into the i-th position of argSizes. | |||
| 1408 | llvm::Value *sizeGEP = builder.CreateInBoundsGEP( | |||
| 1409 | arrI64Ty, mapperAllocas.ArgSizes, | |||
| 1410 | {builder.getInt32(0), builder.getInt32(index)}); | |||
| 1411 | builder.CreateStore(mapOpSize, sizeGEP); | |||
| 1412 | ||||
| 1413 | mapTypeFlags.push_back(mapTypeOp.dyn_cast<mlir::IntegerAttr>().getInt()); | |||
| 1414 | llvm::Constant *mapName = | |||
| 1415 | mlir::LLVM::createMappingInformation(mapOp.getLoc(), *ompBuilder); | |||
| 1416 | mapNames.push_back(mapName); | |||
| 1417 | ++index; | |||
| 1418 | } | |||
| 1419 | ||||
| 1420 | return success(); | |||
| 1421 | } | |||
| 1422 | ||||
| 1423 | static LogicalResult | |||
| 1424 | convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, | |||
| 1425 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1426 | unsigned numMapOperands; | |||
| 1427 | llvm::Value *ifCond = nullptr; | |||
| 1428 | int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF; | |||
| 1429 | SmallVector<Value> mapOperands; | |||
| 1430 | ArrayAttr mapTypes; | |||
| 1431 | ||||
| 1432 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1433 | ||||
| 1434 | LogicalResult result = | |||
| 1435 | llvm::TypeSwitch<Operation *, LogicalResult>(op) | |||
| 1436 | .Case([&](omp::DataOp dataOp) { | |||
| 1437 | if (!dataOp.getUseDeviceAddr().empty() || | |||
| 1438 | !dataOp.getUseDevicePtr().empty()) | |||
| 1439 | return failure(); | |||
| 1440 | ||||
| 1441 | if (auto ifExprVar = dataOp.getIfExpr()) | |||
| 1442 | ifCond = moduleTranslation.lookupValue(ifExprVar); | |||
| 1443 | ||||
| 1444 | if (auto devId = dataOp.getDevice()) | |||
| 1445 | if (auto constOp = mlir::dyn_cast<mlir::LLVM::ConstantOp>( | |||
| 1446 | devId.getDefiningOp())) | |||
| 1447 | if (auto intAttr = | |||
| 1448 | constOp.getValue().dyn_cast<mlir::IntegerAttr>()) | |||
| 1449 | deviceID = intAttr.getInt(); | |||
| 1450 | ||||
| 1451 | numMapOperands = dataOp.getMapOperands().size(); | |||
| 1452 | mapOperands = dataOp.getMapOperands(); | |||
| 1453 | mapTypes = dataOp.getMapTypes(); | |||
| 1454 | return success(); | |||
| 1455 | }) | |||
| 1456 | .Case([&](omp::EnterDataOp enterDataOp) { | |||
| 1457 | if (enterDataOp.getNowait()) | |||
| 1458 | return failure(); | |||
| 1459 | ||||
| 1460 | if (auto ifExprVar = enterDataOp.getIfExpr()) | |||
| 1461 | ifCond = moduleTranslation.lookupValue(ifExprVar); | |||
| 1462 | ||||
| 1463 | if (auto devId = enterDataOp.getDevice()) | |||
| 1464 | if (auto constOp = mlir::dyn_cast<mlir::LLVM::ConstantOp>( | |||
| 1465 | devId.getDefiningOp())) | |||
| 1466 | if (auto intAttr = | |||
| 1467 | constOp.getValue().dyn_cast<mlir::IntegerAttr>()) | |||
| 1468 | deviceID = intAttr.getInt(); | |||
| 1469 | ||||
| 1470 | numMapOperands = enterDataOp.getMapOperands().size(); | |||
| 1471 | mapOperands = enterDataOp.getMapOperands(); | |||
| 1472 | mapTypes = enterDataOp.getMapTypes(); | |||
| 1473 | return success(); | |||
| 1474 | }) | |||
| 1475 | .Case([&](omp::ExitDataOp exitDataOp) { | |||
| 1476 | if (exitDataOp.getNowait()) | |||
| 1477 | return failure(); | |||
| 1478 | ||||
| 1479 | if (auto ifExprVar = exitDataOp.getIfExpr()) | |||
| 1480 | ifCond = moduleTranslation.lookupValue(ifExprVar); | |||
| 1481 | ||||
| 1482 | if (auto devId = exitDataOp.getDevice()) | |||
| 1483 | if (auto constOp = mlir::dyn_cast<mlir::LLVM::ConstantOp>( | |||
| 1484 | devId.getDefiningOp())) | |||
| 1485 | if (auto intAttr = | |||
| 1486 | constOp.getValue().dyn_cast<mlir::IntegerAttr>()) | |||
| 1487 | deviceID = intAttr.getInt(); | |||
| 1488 | ||||
| 1489 | numMapOperands = exitDataOp.getMapOperands().size(); | |||
| 1490 | mapOperands = exitDataOp.getMapOperands(); | |||
| 1491 | mapTypes = exitDataOp.getMapTypes(); | |||
| 1492 | return success(); | |||
| 1493 | }) | |||
| 1494 | .Default([&](Operation *op) { | |||
| 1495 | return op->emitError("unsupported OpenMP operation: ") | |||
| 1496 | << op->getName(); | |||
| 1497 | }); | |||
| 1498 | ||||
| 1499 | if (failed(result)) | |||
| 1500 | return failure(); | |||
| 1501 | ||||
| 1502 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1503 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = | |||
| 1504 | findAllocaInsertPoint(builder, moduleTranslation); | |||
| 1505 | ||||
| 1506 | struct llvm::OpenMPIRBuilder::MapperAllocas mapperAllocas; | |||
| 1507 | SmallVector<uint64_t> mapTypeFlags; | |||
| 1508 | SmallVector<llvm::Constant *> mapNames; | |||
| 1509 | ompBuilder->createMapperAllocas(builder.saveIP(), allocaIP, numMapOperands, | |||
| 1510 | mapperAllocas); | |||
| 1511 | ||||
| 1512 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 1513 | LogicalResult processMapOpStatus = success(); | |||
| 1514 | auto processMapOpCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 1515 | builder.restoreIP(codeGenIP); | |||
| 1516 | processMapOpStatus = | |||
| 1517 | processMapOperand(builder, moduleTranslation, mapOperands, mapTypes, | |||
| 1518 | mapTypeFlags, mapNames, mapperAllocas); | |||
| 1519 | }; | |||
| 1520 | ||||
| 1521 | LogicalResult bodyGenStatus = success(); | |||
| 1522 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { | |||
| 1523 | // DataOp has only one region associated with it. | |||
| 1524 | auto ®ion = cast<omp::DataOp>(op).getRegion(); | |||
| 1525 | builder.restoreIP(codeGenIP); | |||
| 1526 | bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region", builder, | |||
| 1527 | moduleTranslation); | |||
| 1528 | }; | |||
| 1529 | ||||
| 1530 | if (isa<omp::DataOp>(op)) { | |||
| 1531 | builder.restoreIP(ompBuilder->createTargetData( | |||
| 1532 | ompLoc, builder.saveIP(), mapTypeFlags, mapNames, mapperAllocas, | |||
| 1533 | /*IsBegin=*/false, deviceID, ifCond, processMapOpCB, bodyCB)); | |||
| 1534 | } else { | |||
| 1535 | builder.restoreIP(ompBuilder->createTargetData( | |||
| 1536 | ompLoc, builder.saveIP(), mapTypeFlags, mapNames, mapperAllocas, | |||
| 1537 | isa<omp::EnterDataOp>(op), deviceID, ifCond, processMapOpCB)); | |||
| 1538 | } | |||
| 1539 | ||||
| 1540 | if (failed(processMapOpStatus)) | |||
| 1541 | return processMapOpStatus; | |||
| 1542 | return bodyGenStatus; | |||
| 1543 | } | |||
| 1544 | ||||
| 1545 | /// Lowers the FlagsAttr which is applied to the module on the device | |||
| 1546 | /// pass when offloading, this attribute contains OpenMP RTL globals that can | |||
| 1547 | /// be passed as flags to the frontend, otherwise they are set to default | |||
| 1548 | LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute, | |||
| 1549 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1550 | if (!cast<mlir::ModuleOp>(op)) | |||
| 1551 | return failure(); | |||
| 1552 | ||||
| 1553 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1554 | ||||
| 1555 | ompBuilder->createGlobalFlag( | |||
| 1556 | attribute.getDebugKind() /*LangOpts().OpenMPTargetDebug*/, | |||
| 1557 | "__omp_rtl_debug_kind"); | |||
| 1558 | ompBuilder->createGlobalFlag( | |||
| 1559 | attribute | |||
| 1560 | .getAssumeTeamsOversubscription() /*LangOpts().OpenMPTeamSubscription*/ | |||
| 1561 | , | |||
| 1562 | "__omp_rtl_assume_teams_oversubscription"); | |||
| 1563 | ompBuilder->createGlobalFlag( | |||
| 1564 | attribute | |||
| 1565 | .getAssumeThreadsOversubscription() /*LangOpts().OpenMPThreadSubscription*/ | |||
| 1566 | , | |||
| 1567 | "__omp_rtl_assume_threads_oversubscription"); | |||
| 1568 | ompBuilder->createGlobalFlag( | |||
| 1569 | attribute.getAssumeNoThreadState() /*LangOpts().OpenMPNoThreadState*/, | |||
| 1570 | "__omp_rtl_assume_no_thread_state"); | |||
| 1571 | ompBuilder->createGlobalFlag( | |||
| 1572 | attribute | |||
| 1573 | .getAssumeNoNestedParallelism() /*LangOpts().OpenMPNoNestedParallelism*/ | |||
| 1574 | , | |||
| 1575 | "__omp_rtl_assume_no_nested_parallelism"); | |||
| 1576 | ||||
| 1577 | return success(); | |||
| 1578 | } | |||
| 1579 | ||||
| 1580 | static bool getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo, | |||
| 1581 | omp::TargetOp targetOp, | |||
| 1582 | llvm::StringRef parentName = "") { | |||
| 1583 | auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>(); | |||
| 1584 | ||||
| 1585 | assert(fileLoc && "No file found from location")(static_cast <bool> (fileLoc && "No file found from location" ) ? void (0) : __assert_fail ("fileLoc && \"No file found from location\"" , "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp" , 1585, __extension__ __PRETTY_FUNCTION__)); | |||
| 1586 | StringRef fileName = fileLoc.getFilename().getValue(); | |||
| 1587 | ||||
| 1588 | llvm::sys::fs::UniqueID id; | |||
| 1589 | if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) { | |||
| 1590 | targetOp.emitError("Unable to get unique ID for file"); | |||
| 1591 | return false; | |||
| 1592 | } | |||
| 1593 | ||||
| 1594 | uint64_t line = fileLoc.getLine(); | |||
| 1595 | targetInfo = llvm::TargetRegionEntryInfo(parentName, id.getDevice(), | |||
| 1596 | id.getFile(), line); | |||
| 1597 | return true; | |||
| 1598 | } | |||
| 1599 | ||||
| 1600 | static bool targetOpSupported(Operation &opInst) { | |||
| 1601 | auto targetOp = cast<omp::TargetOp>(opInst); | |||
| 1602 | if (targetOp.getIfExpr()) { | |||
| 1603 | opInst.emitError("If clause not yet supported"); | |||
| 1604 | return false; | |||
| 1605 | } | |||
| 1606 | ||||
| 1607 | if (targetOp.getDevice()) { | |||
| 1608 | opInst.emitError("Device clause not yet supported"); | |||
| 1609 | return false; | |||
| 1610 | } | |||
| 1611 | ||||
| 1612 | if (targetOp.getThreadLimit()) { | |||
| 1613 | opInst.emitError("Thread limit clause not yet supported"); | |||
| 1614 | return false; | |||
| 1615 | } | |||
| 1616 | ||||
| 1617 | if (targetOp.getNowait()) { | |||
| 1618 | opInst.emitError("Nowait clause not yet supported"); | |||
| 1619 | return false; | |||
| 1620 | } | |||
| 1621 | ||||
| 1622 | return true; | |||
| 1623 | } | |||
| 1624 | ||||
| 1625 | static LogicalResult | |||
| 1626 | convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, | |||
| 1627 | LLVM::ModuleTranslation &moduleTranslation) { | |||
| 1628 | ||||
| 1629 | if (!targetOpSupported(opInst)) | |||
| 1630 | return failure(); | |||
| 1631 | ||||
| 1632 | bool isDevice = false; | |||
| 1633 | if (auto offloadMod = dyn_cast<mlir::omp::OffloadModuleInterface>( | |||
| 1634 | opInst.getParentOfType<mlir::ModuleOp>().getOperation())) { | |||
| 1635 | isDevice = offloadMod.getIsDevice(); | |||
| 1636 | } | |||
| 1637 | ||||
| 1638 | if (isDevice) // TODO: Implement device codegen. | |||
| 1639 | return success(); | |||
| 1640 | ||||
| 1641 | auto targetOp = cast<omp::TargetOp>(opInst); | |||
| 1642 | auto &targetRegion = targetOp.getRegion(); | |||
| 1643 | ||||
| 1644 | llvm::SetVector<Value> operandSet; | |||
| 1645 | getUsedValuesDefinedAbove(targetRegion, operandSet); | |||
| 1646 | ||||
| 1647 | // Collect the input arguments. | |||
| 1648 | llvm::SmallVector<llvm::Value *> inputs; | |||
| 1649 | for (Value operand : operandSet) | |||
| 1650 | inputs.push_back(moduleTranslation.lookupValue(operand)); | |||
| 1651 | ||||
| 1652 | LogicalResult bodyGenStatus = success(); | |||
| 1653 | ||||
| 1654 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |||
| 1655 | auto bodyCB = [&](InsertPointTy allocaIP, | |||
| 1656 | InsertPointTy codeGenIP) -> InsertPointTy { | |||
| 1657 | builder.restoreIP(codeGenIP); | |||
| 1658 | llvm::BasicBlock *exitBlock = convertOmpOpRegions( | |||
| ||||
| 1659 | targetRegion, "omp.target", builder, moduleTranslation, bodyGenStatus); | |||
| 1660 | builder.SetInsertPoint(exitBlock); | |||
| 1661 | return builder.saveIP(); | |||
| 1662 | }; | |||
| 1663 | ||||
| 1664 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); | |||
| 1665 | StringRef parentName = opInst.getParentOfType<LLVM::LLVMFuncOp>().getName(); | |||
| 1666 | llvm::TargetRegionEntryInfo entryInfo; | |||
| 1667 | ||||
| 1668 | if (!getTargetEntryUniqueInfo(entryInfo, targetOp, parentName)) | |||
| 1669 | return failure(); | |||
| 1670 | ||||
| 1671 | int32_t defaultValTeams = -1; | |||
| 1672 | int32_t defaultValThreads = -1; | |||
| 1673 | ||||
| 1674 | builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTarget( | |||
| 1675 | ompLoc, builder.saveIP(), entryInfo, defaultValTeams, defaultValThreads, | |||
| 1676 | inputs, bodyCB)); | |||
| 1677 | ||||
| 1678 | return bodyGenStatus; | |||
| 1679 | } | |||
| 1680 | ||||
| 1681 | namespace { | |||
| 1682 | ||||
| 1683 | /// Implementation of the dialect interface that converts operations belonging | |||
| 1684 | /// to the OpenMP dialect to LLVM IR. | |||
| 1685 | class OpenMPDialectLLVMIRTranslationInterface | |||
| 1686 | : public LLVMTranslationDialectInterface { | |||
| 1687 | public: | |||
| 1688 | using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface; | |||
| 1689 | ||||
| 1690 | /// Translates the given operation to LLVM IR using the provided IR builder | |||
| 1691 | /// and saving the state in `moduleTranslation`. | |||
| 1692 | LogicalResult | |||
| 1693 | convertOperation(Operation *op, llvm::IRBuilderBase &builder, | |||
| 1694 | LLVM::ModuleTranslation &moduleTranslation) const final; | |||
| 1695 | ||||
| 1696 | LogicalResult | |||
| 1697 | amendOperation(Operation *op, NamedAttribute attribute, | |||
| 1698 | LLVM::ModuleTranslation &moduleTranslation) const final; | |||
| 1699 | }; | |||
| 1700 | ||||
| 1701 | } // namespace | |||
| 1702 | ||||
| 1703 | /// Given an OpenMP MLIR attribute, create the corresponding LLVM-IR, runtime | |||
| 1704 | /// calls, or operation amendments | |||
| 1705 | LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation( | |||
| 1706 | Operation *op, NamedAttribute attribute, | |||
| 1707 | LLVM::ModuleTranslation &moduleTranslation) const { | |||
| 1708 | ||||
| 1709 | return llvm::TypeSwitch<Attribute, LogicalResult>(attribute.getValue()) | |||
| 1710 | .Case([&](mlir::omp::FlagsAttr rtlAttr) { | |||
| 1711 | return convertFlagsAttr(op, rtlAttr, moduleTranslation); | |||
| 1712 | }) | |||
| 1713 | .Default([&](Attribute attr) { | |||
| 1714 | // fall through for omp attributes that do not require lowering and/or | |||
| 1715 | // have no concrete definition and thus no type to define a case on | |||
| 1716 | return success(); | |||
| 1717 | }); | |||
| 1718 | ||||
| 1719 | return failure(); | |||
| 1720 | } | |||
| 1721 | ||||
| 1722 | /// Given an OpenMP MLIR operation, create the corresponding LLVM IR | |||
| 1723 | /// (including OpenMP runtime calls). | |||
| 1724 | LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation( | |||
| 1725 | Operation *op, llvm::IRBuilderBase &builder, | |||
| 1726 | LLVM::ModuleTranslation &moduleTranslation) const { | |||
| 1727 | ||||
| 1728 | llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); | |||
| 1729 | ||||
| 1730 | return llvm::TypeSwitch<Operation *, LogicalResult>(op) | |||
| 1731 | .Case([&](omp::BarrierOp) { | |||
| 1732 | ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier); | |||
| 1733 | return success(); | |||
| 1734 | }) | |||
| 1735 | .Case([&](omp::TaskwaitOp) { | |||
| 1736 | ompBuilder->createTaskwait(builder.saveIP()); | |||
| 1737 | return success(); | |||
| 1738 | }) | |||
| 1739 | .Case([&](omp::TaskyieldOp) { | |||
| 1740 | ompBuilder->createTaskyield(builder.saveIP()); | |||
| 1741 | return success(); | |||
| 1742 | }) | |||
| 1743 | .Case([&](omp::FlushOp) { | |||
| 1744 | // No support in Openmp runtime function (__kmpc_flush) to accept | |||
| 1745 | // the argument list. | |||
| 1746 | // OpenMP standard states the following: | |||
| 1747 | // "An implementation may implement a flush with a list by ignoring | |||
| 1748 | // the list, and treating it the same as a flush without a list." | |||
| 1749 | // | |||
| 1750 | // The argument list is discarded so that, flush with a list is treated | |||
| 1751 | // same as a flush without a list. | |||
| 1752 | ompBuilder->createFlush(builder.saveIP()); | |||
| 1753 | return success(); | |||
| 1754 | }) | |||
| 1755 | .Case([&](omp::ParallelOp op) { | |||
| 1756 | return convertOmpParallel(op, builder, moduleTranslation); | |||
| 1757 | }) | |||
| 1758 | .Case([&](omp::ReductionOp reductionOp) { | |||
| 1759 | return convertOmpReductionOp(reductionOp, builder, moduleTranslation); | |||
| 1760 | }) | |||
| 1761 | .Case([&](omp::MasterOp) { | |||
| 1762 | return convertOmpMaster(*op, builder, moduleTranslation); | |||
| 1763 | }) | |||
| 1764 | .Case([&](omp::CriticalOp) { | |||
| 1765 | return convertOmpCritical(*op, builder, moduleTranslation); | |||
| 1766 | }) | |||
| 1767 | .Case([&](omp::OrderedRegionOp) { | |||
| 1768 | return convertOmpOrderedRegion(*op, builder, moduleTranslation); | |||
| 1769 | }) | |||
| 1770 | .Case([&](omp::OrderedOp) { | |||
| 1771 | return convertOmpOrdered(*op, builder, moduleTranslation); | |||
| 1772 | }) | |||
| 1773 | .Case([&](omp::WsLoopOp) { | |||
| 1774 | return convertOmpWsLoop(*op, builder, moduleTranslation); | |||
| 1775 | }) | |||
| 1776 | .Case([&](omp::SimdLoopOp) { | |||
| 1777 | return convertOmpSimdLoop(*op, builder, moduleTranslation); | |||
| 1778 | }) | |||
| 1779 | .Case([&](omp::AtomicReadOp) { | |||
| 1780 | return convertOmpAtomicRead(*op, builder, moduleTranslation); | |||
| 1781 | }) | |||
| 1782 | .Case([&](omp::AtomicWriteOp) { | |||
| 1783 | return convertOmpAtomicWrite(*op, builder, moduleTranslation); | |||
| 1784 | }) | |||
| 1785 | .Case([&](omp::AtomicUpdateOp op) { | |||
| 1786 | return convertOmpAtomicUpdate(op, builder, moduleTranslation); | |||
| 1787 | }) | |||
| 1788 | .Case([&](omp::AtomicCaptureOp op) { | |||
| 1789 | return convertOmpAtomicCapture(op, builder, moduleTranslation); | |||
| 1790 | }) | |||
| 1791 | .Case([&](omp::SectionsOp) { | |||
| 1792 | return convertOmpSections(*op, builder, moduleTranslation); | |||
| 1793 | }) | |||
| 1794 | .Case([&](omp::SingleOp op) { | |||
| 1795 | return convertOmpSingle(op, builder, moduleTranslation); | |||
| 1796 | }) | |||
| 1797 | .Case([&](omp::TaskOp op) { | |||
| 1798 | return convertOmpTaskOp(op, builder, moduleTranslation); | |||
| 1799 | }) | |||
| 1800 | .Case([&](omp::TaskGroupOp op) { | |||
| 1801 | return convertOmpTaskgroupOp(op, builder, moduleTranslation); | |||
| 1802 | }) | |||
| 1803 | .Case<omp::YieldOp, omp::TerminatorOp, omp::ReductionDeclareOp, | |||
| 1804 | omp::CriticalDeclareOp>([](auto op) { | |||
| 1805 | // `yield` and `terminator` can be just omitted. The block structure | |||
| 1806 | // was created in the region that handles their parent operation. | |||
| 1807 | // `reduction.declare` will be used by reductions and is not | |||
| 1808 | // converted directly, skip it. | |||
| 1809 | // `critical.declare` is only used to declare names of critical | |||
| 1810 | // sections which will be used by `critical` ops and hence can be | |||
| 1811 | // ignored for lowering. The OpenMP IRBuilder will create unique | |||
| 1812 | // name for critical section names. | |||
| 1813 | return success(); | |||
| 1814 | }) | |||
| 1815 | .Case([&](omp::ThreadprivateOp) { | |||
| 1816 | return convertOmpThreadprivate(*op, builder, moduleTranslation); | |||
| 1817 | }) | |||
| 1818 | .Case<omp::DataOp, omp::EnterDataOp, omp::ExitDataOp>([&](auto op) { | |||
| 1819 | return convertOmpTargetData(op, builder, moduleTranslation); | |||
| 1820 | }) | |||
| 1821 | .Case([&](omp::TargetOp) { | |||
| 1822 | return convertOmpTarget(*op, builder, moduleTranslation); | |||
| 1823 | }) | |||
| 1824 | .Default([&](Operation *inst) { | |||
| 1825 | return inst->emitError("unsupported OpenMP operation: ") | |||
| 1826 | << inst->getName(); | |||
| 1827 | }); | |||
| 1828 | } | |||
| 1829 | ||||
| 1830 | void mlir::registerOpenMPDialectTranslation(DialectRegistry ®istry) { | |||
| 1831 | registry.insert<omp::OpenMPDialect>(); | |||
| 1832 | registry.addExtension(+[](MLIRContext *ctx, omp::OpenMPDialect *dialect) { | |||
| 1833 | dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>(); | |||
| 1834 | }); | |||
| 1835 | } | |||
| 1836 | ||||
| 1837 | void mlir::registerOpenMPDialectTranslation(MLIRContext &context) { | |||
| 1838 | DialectRegistry registry; | |||
| 1839 | registerOpenMPDialectTranslation(registry); | |||
| 1840 | context.appendDialectRegistry(registry); | |||
| 1841 | } |