Bug Summary

File:build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
Warning:line 218, column 9
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name OpenMPToLLVMIRTranslation.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D MLIR_CUDA_CONVERSIONS_ENABLED=1 -D MLIR_ROCM_CONVERSIONS_ENABLED=1 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I tools/mlir/lib/Target/LLVMIR/Dialect/OpenMP -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/mlir/lib/Target/LLVMIR/Dialect/OpenMP -I include -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/mlir/include -I tools/mlir/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-09-04-125545-48738-1 -x c++ /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
1//===- OpenMPToLLVMIRTranslation.cpp - Translate OpenMP dialect to LLVM IR-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a translation between the MLIR OpenMP dialect and LLVM
10// IR.
11//
12//===----------------------------------------------------------------------===//
13#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
14#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
15#include "mlir/IR/BlockAndValueMapping.h"
16#include "mlir/IR/Operation.h"
17#include "mlir/Support/LLVM.h"
18#include "mlir/Target/LLVMIR/ModuleTranslation.h"
19
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/TypeSwitch.h"
22#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
23#include "llvm/IR/DebugInfoMetadata.h"
24#include "llvm/IR/IRBuilder.h"
25
26using namespace mlir;
27
28namespace {
29static llvm::omp::ScheduleKind
30convertToScheduleKind(Optional<omp::ClauseScheduleKind> schedKind) {
31 if (!schedKind.has_value())
32 return llvm::omp::OMP_SCHEDULE_Default;
33 switch (schedKind.value()) {
34 case omp::ClauseScheduleKind::Static:
35 return llvm::omp::OMP_SCHEDULE_Static;
36 case omp::ClauseScheduleKind::Dynamic:
37 return llvm::omp::OMP_SCHEDULE_Dynamic;
38 case omp::ClauseScheduleKind::Guided:
39 return llvm::omp::OMP_SCHEDULE_Guided;
40 case omp::ClauseScheduleKind::Auto:
41 return llvm::omp::OMP_SCHEDULE_Auto;
42 case omp::ClauseScheduleKind::Runtime:
43 return llvm::omp::OMP_SCHEDULE_Runtime;
44 }
45 llvm_unreachable("unhandled schedule clause argument")::llvm::llvm_unreachable_internal("unhandled schedule clause argument"
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 45)
;
46}
47
48/// ModuleTranslation stack frame for OpenMP operations. This keeps track of the
49/// insertion points for allocas.
50class OpenMPAllocaStackFrame
51 : public LLVM::ModuleTranslation::StackFrameBase<OpenMPAllocaStackFrame> {
52public:
53 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocaStackFrame)static ::mlir::TypeID resolveTypeID() { static ::mlir::SelfOwningTypeID
id; return id; } static_assert( ::mlir::detail::InlineTypeIDResolver
::has_resolve_typeid< OpenMPAllocaStackFrame>::value, "`MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID` must be placed in a "
"public section of `" "OpenMPAllocaStackFrame" "`");
54
55 explicit OpenMPAllocaStackFrame(llvm::OpenMPIRBuilder::InsertPointTy allocaIP)
56 : allocaInsertPoint(allocaIP) {}
57 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
58};
59
60/// ModuleTranslation stack frame containing the partial mapping between MLIR
61/// values and their LLVM IR equivalents.
62class OpenMPVarMappingStackFrame
63 : public LLVM::ModuleTranslation::StackFrameBase<
64 OpenMPVarMappingStackFrame> {
65public:
66 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPVarMappingStackFrame)static ::mlir::TypeID resolveTypeID() { static ::mlir::SelfOwningTypeID
id; return id; } static_assert( ::mlir::detail::InlineTypeIDResolver
::has_resolve_typeid< OpenMPVarMappingStackFrame>::value
, "`MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID` must be placed in a "
"public section of `" "OpenMPVarMappingStackFrame" "`");
67
68 explicit OpenMPVarMappingStackFrame(
69 const DenseMap<Value, llvm::Value *> &mapping)
70 : mapping(mapping) {}
71
72 DenseMap<Value, llvm::Value *> mapping;
73};
74} // namespace
75
76/// Find the insertion point for allocas given the current insertion point for
77/// normal operations in the builder.
78static llvm::OpenMPIRBuilder::InsertPointTy
79findAllocaInsertPoint(llvm::IRBuilderBase &builder,
80 const LLVM::ModuleTranslation &moduleTranslation) {
81 // If there is an alloca insertion point on stack, i.e. we are in a nested
82 // operation and a specific point was provided by some surrounding operation,
83 // use it.
84 llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
85 WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocaStackFrame>(
86 [&](const OpenMPAllocaStackFrame &frame) {
87 allocaInsertPoint = frame.allocaInsertPoint;
88 return WalkResult::interrupt();
89 });
90 if (walkResult.wasInterrupted())
91 return allocaInsertPoint;
92
93 // Otherwise, insert to the entry block of the surrounding function.
94 // If the current IRBuilder InsertPoint is the function's entry, it cannot
95 // also be used for alloca insertion which would result in insertion order
96 // confusion. Create a new BasicBlock for the Builder and use the entry block
97 // for the allocs.
98 // TODO: Create a dedicated alloca BasicBlock at function creation such that
99 // we do not need to move the current InertPoint here.
100 if (builder.GetInsertBlock() ==
101 &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
102 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&(static_cast <bool> (builder.GetInsertPoint() == builder
.GetInsertBlock()->end() && "Assuming end of basic block"
) ? void (0) : __assert_fail ("builder.GetInsertPoint() == builder.GetInsertBlock()->end() && \"Assuming end of basic block\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 103, __extension__ __PRETTY_FUNCTION__))
103 "Assuming end of basic block")(static_cast <bool> (builder.GetInsertPoint() == builder
.GetInsertBlock()->end() && "Assuming end of basic block"
) ? void (0) : __assert_fail ("builder.GetInsertPoint() == builder.GetInsertBlock()->end() && \"Assuming end of basic block\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 103, __extension__ __PRETTY_FUNCTION__))
;
104 llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
105 builder.getContext(), "entry", builder.GetInsertBlock()->getParent(),
106 builder.GetInsertBlock()->getNextNode());
107 builder.CreateBr(entryBB);
108 builder.SetInsertPoint(entryBB);
109 }
110
111 llvm::BasicBlock &funcEntryBlock =
112 builder.GetInsertBlock()->getParent()->getEntryBlock();
113 return llvm::OpenMPIRBuilder::InsertPointTy(
114 &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
115}
116
117/// Converts the given region that appears within an OpenMP dialect operation to
118/// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the
119/// region, and a branch from any block with an successor-less OpenMP terminator
120/// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes
121/// of the continuation block if provided.
122static llvm::BasicBlock *convertOmpOpRegions(
123 Region &region, StringRef blockName, llvm::IRBuilderBase &builder,
124 LLVM::ModuleTranslation &moduleTranslation, LogicalResult &bodyGenStatus,
125 SmallVectorImpl<llvm::PHINode *> *continuationBlockPHIs = nullptr) {
126 llvm::BasicBlock *continuationBlock =
127 splitBB(builder, true, "omp.region.cont");
128 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
129
130 llvm::LLVMContext &llvmContext = builder.getContext();
131 for (Block &bb : region) {
132 llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(
133 llvmContext, blockName, builder.GetInsertBlock()->getParent(),
134 builder.GetInsertBlock()->getNextNode());
135 moduleTranslation.mapBlock(&bb, llvmBB);
136 }
137
138 llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();
139
140 // Terminators (namely YieldOp) may be forwarding values to the region that
141 // need to be available in the continuation block. Collect the types of these
142 // operands in preparation of creating PHI nodes.
143 SmallVector<llvm::Type *> continuationBlockPHITypes;
144 bool operandsProcessed = false;
145 unsigned numYields = 0;
146 for (Block &bb : region.getBlocks()) {
147 if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(bb.getTerminator())) {
148 if (!operandsProcessed) {
149 for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
150 continuationBlockPHITypes.push_back(
151 moduleTranslation.convertType(yield->getOperand(i).getType()));
152 }
153 operandsProcessed = true;
154 } else {
155 assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&(static_cast <bool> (continuationBlockPHITypes.size() ==
yield->getNumOperands() && "mismatching number of values yielded from the region"
) ? void (0) : __assert_fail ("continuationBlockPHITypes.size() == yield->getNumOperands() && \"mismatching number of values yielded from the region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 156, __extension__ __PRETTY_FUNCTION__))
156 "mismatching number of values yielded from the region")(static_cast <bool> (continuationBlockPHITypes.size() ==
yield->getNumOperands() && "mismatching number of values yielded from the region"
) ? void (0) : __assert_fail ("continuationBlockPHITypes.size() == yield->getNumOperands() && \"mismatching number of values yielded from the region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 156, __extension__ __PRETTY_FUNCTION__))
;
157 for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
158 llvm::Type *operandType =
159 moduleTranslation.convertType(yield->getOperand(i).getType());
160 (void)operandType;
161 assert(continuationBlockPHITypes[i] == operandType &&(static_cast <bool> (continuationBlockPHITypes[i] == operandType
&& "values of mismatching types yielded from the region"
) ? void (0) : __assert_fail ("continuationBlockPHITypes[i] == operandType && \"values of mismatching types yielded from the region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 162, __extension__ __PRETTY_FUNCTION__))
162 "values of mismatching types yielded from the region")(static_cast <bool> (continuationBlockPHITypes[i] == operandType
&& "values of mismatching types yielded from the region"
) ? void (0) : __assert_fail ("continuationBlockPHITypes[i] == operandType && \"values of mismatching types yielded from the region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 162, __extension__ __PRETTY_FUNCTION__))
;
163 }
164 }
165 numYields++;
166 }
167 }
168
169 // Insert PHI nodes in the continuation block for any values forwarded by the
170 // terminators in this region.
171 if (!continuationBlockPHITypes.empty())
5
Taking false branch
172 assert((static_cast <bool> (continuationBlockPHIs && "expected continuation block PHIs if converted regions yield values"
) ? void (0) : __assert_fail ("continuationBlockPHIs && \"expected continuation block PHIs if converted regions yield values\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 174, __extension__ __PRETTY_FUNCTION__))
173 continuationBlockPHIs &&(static_cast <bool> (continuationBlockPHIs && "expected continuation block PHIs if converted regions yield values"
) ? void (0) : __assert_fail ("continuationBlockPHIs && \"expected continuation block PHIs if converted regions yield values\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 174, __extension__ __PRETTY_FUNCTION__))
174 "expected continuation block PHIs if converted regions yield values")(static_cast <bool> (continuationBlockPHIs && "expected continuation block PHIs if converted regions yield values"
) ? void (0) : __assert_fail ("continuationBlockPHIs && \"expected continuation block PHIs if converted regions yield values\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 174, __extension__ __PRETTY_FUNCTION__))
;
175 if (continuationBlockPHIs
5.1
'continuationBlockPHIs' is null
) {
6
Taking false branch
176 llvm::IRBuilderBase::InsertPointGuard guard(builder);
177 continuationBlockPHIs->reserve(continuationBlockPHITypes.size());
178 builder.SetInsertPoint(continuationBlock, continuationBlock->begin());
179 for (llvm::Type *ty : continuationBlockPHITypes)
180 continuationBlockPHIs->push_back(builder.CreatePHI(ty, numYields));
181 }
182
183 // Convert blocks one by one in topological order to ensure
184 // defs are converted before uses.
185 SetVector<Block *> blocks =
186 LLVM::detail::getTopologicallySortedBlocks(region);
187 for (Block *bb : blocks) {
188 llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(bb);
189 // Retarget the branch of the entry block to the entry block of the
190 // converted region (regions are single-entry).
191 if (bb->isEntryBlock()) {
7
Assuming the condition is false
8
Taking false branch
192 assert(sourceTerminator->getNumSuccessors() == 1 &&(static_cast <bool> (sourceTerminator->getNumSuccessors
() == 1 && "provided entry block has multiple successors"
) ? void (0) : __assert_fail ("sourceTerminator->getNumSuccessors() == 1 && \"provided entry block has multiple successors\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 193, __extension__ __PRETTY_FUNCTION__))
193 "provided entry block has multiple successors")(static_cast <bool> (sourceTerminator->getNumSuccessors
() == 1 && "provided entry block has multiple successors"
) ? void (0) : __assert_fail ("sourceTerminator->getNumSuccessors() == 1 && \"provided entry block has multiple successors\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 193, __extension__ __PRETTY_FUNCTION__))
;
194 assert(sourceTerminator->getSuccessor(0) == continuationBlock &&(static_cast <bool> (sourceTerminator->getSuccessor(
0) == continuationBlock && "ContinuationBlock is not the successor of the entry block"
) ? void (0) : __assert_fail ("sourceTerminator->getSuccessor(0) == continuationBlock && \"ContinuationBlock is not the successor of the entry block\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 195, __extension__ __PRETTY_FUNCTION__))
195 "ContinuationBlock is not the successor of the entry block")(static_cast <bool> (sourceTerminator->getSuccessor(
0) == continuationBlock && "ContinuationBlock is not the successor of the entry block"
) ? void (0) : __assert_fail ("sourceTerminator->getSuccessor(0) == continuationBlock && \"ContinuationBlock is not the successor of the entry block\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 195, __extension__ __PRETTY_FUNCTION__))
;
196 sourceTerminator->setSuccessor(0, llvmBB);
197 }
198
199 llvm::IRBuilderBase::InsertPointGuard guard(builder);
200 if (failed(
9
Taking false branch
201 moduleTranslation.convertBlock(*bb, bb->isEntryBlock(), builder))) {
202 bodyGenStatus = failure();
203 return continuationBlock;
204 }
205
206 // Special handling for `omp.yield` and `omp.terminator` (we may have more
207 // than one): they return the control to the parent OpenMP dialect operation
208 // so replace them with the branch to the continuation block. We handle this
209 // here to avoid relying inter-function communication through the
210 // ModuleTranslation class to set up the correct insertion point. This is
211 // also consistent with MLIR's idiom of handling special region terminators
212 // in the same code that handles the region-owning operation.
213 Operation *terminator = bb->getTerminator();
214 if (isa<omp::TerminatorOp, omp::YieldOp>(terminator)) {
10
Assuming 'terminator' is a 'class mlir::omp::YieldOp &'
11
Taking true branch
215 builder.CreateBr(continuationBlock);
216
217 for (unsigned i = 0, e = terminator->getNumOperands(); i < e; ++i)
12
Assuming 'i' is < 'e'
13
Loop condition is true. Entering loop body
218 (*continuationBlockPHIs)[i]->addIncoming(
14
Called C++ object pointer is null
219 moduleTranslation.lookupValue(terminator->getOperand(i)), llvmBB);
220 }
221 }
222 // After all blocks have been traversed and values mapped, connect the PHI
223 // nodes to the results of preceding blocks.
224 LLVM::detail::connectPHINodes(region, moduleTranslation);
225
226 // Remove the blocks and values defined in this region from the mapping since
227 // they are not visible outside of this region. This allows the same region to
228 // be converted several times, that is cloned, without clashes, and slightly
229 // speeds up the lookups.
230 moduleTranslation.forgetMapping(region);
231
232 return continuationBlock;
233}
234
235/// Convert ProcBindKind from MLIR-generated enum to LLVM enum.
236static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind) {
237 switch (kind) {
238 case omp::ClauseProcBindKind::Close:
239 return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;
240 case omp::ClauseProcBindKind::Master:
241 return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;
242 case omp::ClauseProcBindKind::Primary:
243 return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;
244 case omp::ClauseProcBindKind::Spread:
245 return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;
246 }
247 llvm_unreachable("Unknown ClauseProcBindKind kind")::llvm::llvm_unreachable_internal("Unknown ClauseProcBindKind kind"
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 247)
;
248}
249
250/// Converts the OpenMP parallel operation to LLVM IR.
251static LogicalResult
252convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
253 LLVM::ModuleTranslation &moduleTranslation) {
254 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
255 // TODO: support error propagation in OpenMPIRBuilder and use it instead of
256 // relying on captured variables.
257 LogicalResult bodyGenStatus = success();
258
259 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
260 // Save the alloca insertion point on ModuleTranslation stack for use in
261 // nested regions.
262 LLVM::ModuleTranslation::SaveStack<OpenMPAllocaStackFrame> frame(
263 moduleTranslation, allocaIP);
264
265 // ParallelOp has only one region associated with it.
266 builder.restoreIP(codeGenIP);
267 convertOmpOpRegions(opInst.getRegion(), "omp.par.region", builder,
268 moduleTranslation, bodyGenStatus);
269 };
270
271 // TODO: Perform appropriate actions according to the data-sharing
272 // attribute (shared, private, firstprivate, ...) of variables.
273 // Currently defaults to shared.
274 auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
275 llvm::Value &, llvm::Value &vPtr,
276 llvm::Value *&replacementValue) -> InsertPointTy {
277 replacementValue = &vPtr;
278
279 return codeGenIP;
280 };
281
282 // TODO: Perform finalization actions for variables. This has to be
283 // called for variables which have destructors/finalizers.
284 auto finiCB = [&](InsertPointTy codeGenIP) {};
285
286 llvm::Value *ifCond = nullptr;
287 if (auto ifExprVar = opInst.if_expr_var())
288 ifCond = moduleTranslation.lookupValue(ifExprVar);
289 llvm::Value *numThreads = nullptr;
290 if (auto numThreadsVar = opInst.num_threads_var())
291 numThreads = moduleTranslation.lookupValue(numThreadsVar);
292 auto pbKind = llvm::omp::OMP_PROC_BIND_default;
293 if (auto bind = opInst.proc_bind_val())
294 pbKind = getProcBindKind(*bind);
295 // TODO: Is the Parallel construct cancellable?
296 bool isCancellable = false;
297
298 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
299 findAllocaInsertPoint(builder, moduleTranslation);
300 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
301 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createParallel(
302 ompLoc, allocaIP, bodyGenCB, privCB, finiCB, ifCond, numThreads, pbKind,
303 isCancellable));
304
305 return bodyGenStatus;
306}
307
308/// Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.
309static LogicalResult
310convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder,
311 LLVM::ModuleTranslation &moduleTranslation) {
312 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
313 // TODO: support error propagation in OpenMPIRBuilder and use it instead of
314 // relying on captured variables.
315 LogicalResult bodyGenStatus = success();
316
317 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
318 // MasterOp has only one region associated with it.
319 auto &region = cast<omp::MasterOp>(opInst).getRegion();
320 builder.restoreIP(codeGenIP);
321 convertOmpOpRegions(region, "omp.master.region", builder, moduleTranslation,
322 bodyGenStatus);
323 };
324
325 // TODO: Perform finalization actions for variables. This has to be
326 // called for variables which have destructors/finalizers.
327 auto finiCB = [&](InsertPointTy codeGenIP) {};
328
329 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
330 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createMaster(
331 ompLoc, bodyGenCB, finiCB));
332 return success();
333}
334
335/// Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.
336static LogicalResult
337convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder,
338 LLVM::ModuleTranslation &moduleTranslation) {
339 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
340 auto criticalOp = cast<omp::CriticalOp>(opInst);
341 // TODO: support error propagation in OpenMPIRBuilder and use it instead of
342 // relying on captured variables.
343 LogicalResult bodyGenStatus = success();
344
345 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
346 // CriticalOp has only one region associated with it.
347 auto &region = cast<omp::CriticalOp>(opInst).getRegion();
348 builder.restoreIP(codeGenIP);
349 convertOmpOpRegions(region, "omp.critical.region", builder,
350 moduleTranslation, bodyGenStatus);
351 };
352
353 // TODO: Perform finalization actions for variables. This has to be
354 // called for variables which have destructors/finalizers.
355 auto finiCB = [&](InsertPointTy codeGenIP) {};
356
357 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
358 llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();
359 llvm::Constant *hint = nullptr;
360
361 // If it has a name, it probably has a hint too.
362 if (criticalOp.nameAttr()) {
363 // The verifiers in OpenMP Dialect guarentee that all the pointers are
364 // non-null
365 auto symbolRef = criticalOp.nameAttr().cast<SymbolRefAttr>();
366 auto criticalDeclareOp =
367 SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(criticalOp,
368 symbolRef);
369 hint =
370 llvm::ConstantInt::get(llvm::Type::getInt32Ty(llvmContext),
371 static_cast<int>(criticalDeclareOp.hint_val()));
372 }
373 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createCritical(
374 ompLoc, bodyGenCB, finiCB, criticalOp.name().value_or(""), hint));
375 return success();
376}
377
378/// Returns a reduction declaration that corresponds to the given reduction
379/// operation in the given container. Currently only supports reductions inside
380/// WsLoopOp but can be easily extended.
381static omp::ReductionDeclareOp findReductionDecl(omp::WsLoopOp container,
382 omp::ReductionOp reduction) {
383 SymbolRefAttr reductionSymbol;
384 for (unsigned i = 0, e = container.getNumReductionVars(); i < e; ++i) {
385 if (container.reduction_vars()[i] != reduction.accumulator())
386 continue;
387 reductionSymbol = (*container.reductions())[i].cast<SymbolRefAttr>();
388 break;
389 }
390 assert(reductionSymbol &&(static_cast <bool> (reductionSymbol && "reduction operation must be associated with a declaration"
) ? void (0) : __assert_fail ("reductionSymbol && \"reduction operation must be associated with a declaration\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 391, __extension__ __PRETTY_FUNCTION__))
391 "reduction operation must be associated with a declaration")(static_cast <bool> (reductionSymbol && "reduction operation must be associated with a declaration"
) ? void (0) : __assert_fail ("reductionSymbol && \"reduction operation must be associated with a declaration\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 391, __extension__ __PRETTY_FUNCTION__))
;
392
393 return SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>(
394 container, reductionSymbol);
395}
396
397/// Populates `reductions` with reduction declarations used in the given loop.
398static void
399collectReductionDecls(omp::WsLoopOp loop,
400 SmallVectorImpl<omp::ReductionDeclareOp> &reductions) {
401 Optional<ArrayAttr> attr = loop.reductions();
402 if (!attr)
403 return;
404
405 reductions.reserve(reductions.size() + loop.getNumReductionVars());
406 for (auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
407 reductions.push_back(
408 SymbolTable::lookupNearestSymbolFrom<omp::ReductionDeclareOp>(
409 loop, symbolRef));
410 }
411}
412
413/// Translates the blocks contained in the given region and appends them to at
414/// the current insertion point of `builder`. The operations of the entry block
415/// are appended to the current insertion block, which is not expected to have a
416/// terminator. If set, `continuationBlockArgs` is populated with translated
417/// values that correspond to the values omp.yield'ed from the region.
418static LogicalResult inlineConvertOmpRegions(
419 Region &region, StringRef blockName, llvm::IRBuilderBase &builder,
420 LLVM::ModuleTranslation &moduleTranslation,
421 SmallVectorImpl<llvm::Value *> *continuationBlockArgs = nullptr) {
422 if (region.empty())
423 return success();
424
425 // Special case for single-block regions that don't create additional blocks:
426 // insert operations without creating additional blocks.
427 if (llvm::hasSingleElement(region)) {
428 moduleTranslation.mapBlock(&region.front(), builder.GetInsertBlock());
429 if (failed(moduleTranslation.convertBlock(
430 region.front(), /*ignoreArguments=*/true, builder)))
431 return failure();
432
433 // The continuation arguments are simply the translated terminator operands.
434 if (continuationBlockArgs)
435 llvm::append_range(
436 *continuationBlockArgs,
437 moduleTranslation.lookupValues(region.front().back().getOperands()));
438
439 // Drop the mapping that is no longer necessary so that the same region can
440 // be processed multiple times.
441 moduleTranslation.forgetMapping(region);
442 return success();
443 }
444
445 LogicalResult bodyGenStatus = success();
446 SmallVector<llvm::PHINode *> phis;
447 llvm::BasicBlock *continuationBlock = convertOmpOpRegions(
448 region, blockName, builder, moduleTranslation, bodyGenStatus, &phis);
449 if (failed(bodyGenStatus))
450 return failure();
451 if (continuationBlockArgs)
452 llvm::append_range(*continuationBlockArgs, phis);
453 builder.SetInsertPoint(continuationBlock,
454 continuationBlock->getFirstInsertionPt());
455 return success();
456}
457
458namespace {
459/// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to
460/// store lambdas with capture.
461using OwningReductionGen = std::function<llvm::OpenMPIRBuilder::InsertPointTy(
462 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
463 llvm::Value *&)>;
464using OwningAtomicReductionGen =
465 std::function<llvm::OpenMPIRBuilder::InsertPointTy(
466 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
467 llvm::Value *)>;
468} // namespace
469
470/// Create an OpenMPIRBuilder-compatible reduction generator for the given
471/// reduction declaration. The generator uses `builder` but ignores its
472/// insertion point.
473static OwningReductionGen
474makeReductionGen(omp::ReductionDeclareOp decl, llvm::IRBuilderBase &builder,
475 LLVM::ModuleTranslation &moduleTranslation) {
476 // The lambda is mutable because we need access to non-const methods of decl
477 // (which aren't actually mutating it), and we must capture decl by-value to
478 // avoid the dangling reference after the parent function returns.
479 OwningReductionGen gen =
480 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
481 llvm::Value *lhs, llvm::Value *rhs,
482 llvm::Value *&result) mutable {
483 Region &reductionRegion = decl.reductionRegion();
484 moduleTranslation.mapValue(reductionRegion.front().getArgument(0), lhs);
485 moduleTranslation.mapValue(reductionRegion.front().getArgument(1), rhs);
486 builder.restoreIP(insertPoint);
487 SmallVector<llvm::Value *> phis;
488 if (failed(inlineConvertOmpRegions(reductionRegion,
489 "omp.reduction.nonatomic.body",
490 builder, moduleTranslation, &phis)))
491 return llvm::OpenMPIRBuilder::InsertPointTy();
492 assert(phis.size() == 1)(static_cast <bool> (phis.size() == 1) ? void (0) : __assert_fail
("phis.size() == 1", "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 492, __extension__ __PRETTY_FUNCTION__))
;
493 result = phis[0];
494 return builder.saveIP();
495 };
496 return gen;
497}
498
499/// Create an OpenMPIRBuilder-compatible atomic reduction generator for the
500/// given reduction declaration. The generator uses `builder` but ignores its
501/// insertion point. Returns null if there is no atomic region available in the
502/// reduction declaration.
503static OwningAtomicReductionGen
504makeAtomicReductionGen(omp::ReductionDeclareOp decl,
505 llvm::IRBuilderBase &builder,
506 LLVM::ModuleTranslation &moduleTranslation) {
507 if (decl.atomicReductionRegion().empty())
508 return OwningAtomicReductionGen();
509
510 // The lambda is mutable because we need access to non-const methods of decl
511 // (which aren't actually mutating it), and we must capture decl by-value to
512 // avoid the dangling reference after the parent function returns.
513 OwningAtomicReductionGen atomicGen =
514 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
515 llvm::Value *lhs, llvm::Value *rhs) mutable {
516 Region &atomicRegion = decl.atomicReductionRegion();
517 moduleTranslation.mapValue(atomicRegion.front().getArgument(0), lhs);
518 moduleTranslation.mapValue(atomicRegion.front().getArgument(1), rhs);
519 builder.restoreIP(insertPoint);
520 SmallVector<llvm::Value *> phis;
521 if (failed(inlineConvertOmpRegions(atomicRegion,
522 "omp.reduction.atomic.body", builder,
523 moduleTranslation, &phis)))
524 return llvm::OpenMPIRBuilder::InsertPointTy();
525 assert(phis.empty())(static_cast <bool> (phis.empty()) ? void (0) : __assert_fail
("phis.empty()", "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 525, __extension__ __PRETTY_FUNCTION__))
;
526 return builder.saveIP();
527 };
528 return atomicGen;
529}
530
531/// Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.
532static LogicalResult
533convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder,
534 LLVM::ModuleTranslation &moduleTranslation) {
535 auto orderedOp = cast<omp::OrderedOp>(opInst);
536
537 omp::ClauseDepend dependType = *orderedOp.depend_type_val();
538 bool isDependSource = dependType == omp::ClauseDepend::dependsource;
539 unsigned numLoops = *orderedOp.num_loops_val();
540 SmallVector<llvm::Value *> vecValues =
541 moduleTranslation.lookupValues(orderedOp.depend_vec_vars());
542
543 size_t indexVecValues = 0;
544 while (indexVecValues < vecValues.size()) {
545 SmallVector<llvm::Value *> storeValues;
546 storeValues.reserve(numLoops);
547 for (unsigned i = 0; i < numLoops; i++) {
548 storeValues.push_back(vecValues[indexVecValues]);
549 indexVecValues++;
550 }
551 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
552 findAllocaInsertPoint(builder, moduleTranslation);
553 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
554 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createOrderedDepend(
555 ompLoc, allocaIP, numLoops, storeValues, ".cnt.addr", isDependSource));
556 }
557 return success();
558}
559
560/// Converts an OpenMP 'ordered_region' operation into LLVM IR using
561/// OpenMPIRBuilder.
562static LogicalResult
563convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder,
564 LLVM::ModuleTranslation &moduleTranslation) {
565 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
566 auto orderedRegionOp = cast<omp::OrderedRegionOp>(opInst);
567
568 // TODO: The code generation for ordered simd directive is not supported yet.
569 if (orderedRegionOp.simd())
570 return failure();
571
572 // TODO: support error propagation in OpenMPIRBuilder and use it instead of
573 // relying on captured variables.
574 LogicalResult bodyGenStatus = success();
575
576 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
577 // OrderedOp has only one region associated with it.
578 auto &region = cast<omp::OrderedRegionOp>(opInst).getRegion();
579 builder.restoreIP(codeGenIP);
580 convertOmpOpRegions(region, "omp.ordered.region", builder,
581 moduleTranslation, bodyGenStatus);
582 };
583
584 // TODO: Perform finalization actions for variables. This has to be
585 // called for variables which have destructors/finalizers.
586 auto finiCB = [&](InsertPointTy codeGenIP) {};
587
588 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
589 builder.restoreIP(
590 moduleTranslation.getOpenMPBuilder()->createOrderedThreadsSimd(
591 ompLoc, bodyGenCB, finiCB, !orderedRegionOp.simd()));
592 return bodyGenStatus;
593}
594
595static LogicalResult
596convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
597 LLVM::ModuleTranslation &moduleTranslation) {
598 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
599 using StorableBodyGenCallbackTy =
600 llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
601
602 auto sectionsOp = cast<omp::SectionsOp>(opInst);
603
604 // TODO: Support the following clauses: private, firstprivate, lastprivate,
605 // reduction, allocate
606 if (!sectionsOp.reduction_vars().empty() || sectionsOp.reductions() ||
607 !sectionsOp.allocate_vars().empty() ||
608 !sectionsOp.allocators_vars().empty())
609 return emitError(sectionsOp.getLoc())
610 << "reduction and allocate clauses are not supported for sections "
611 "construct";
612
613 LogicalResult bodyGenStatus = success();
614 SmallVector<StorableBodyGenCallbackTy> sectionCBs;
615
616 for (Operation &op : *sectionsOp.region().begin()) {
617 auto sectionOp = dyn_cast<omp::SectionOp>(op);
618 if (!sectionOp) // omp.terminator
619 continue;
620
621 Region &region = sectionOp.region();
622 auto sectionCB = [&region, &builder, &moduleTranslation, &bodyGenStatus](
623 InsertPointTy allocaIP, InsertPointTy codeGenIP) {
624 builder.restoreIP(codeGenIP);
625 convertOmpOpRegions(region, "omp.section.region", builder,
626 moduleTranslation, bodyGenStatus);
627 };
628 sectionCBs.push_back(sectionCB);
629 }
630
631 // No sections within omp.sections operation - skip generation. This situation
632 // is only possible if there is only a terminator operation inside the
633 // sections operation
634 if (sectionCBs.empty())
635 return success();
636
637 assert(isa<omp::SectionOp>(*sectionsOp.region().op_begin()))(static_cast <bool> (isa<omp::SectionOp>(*sectionsOp
.region().op_begin())) ? void (0) : __assert_fail ("isa<omp::SectionOp>(*sectionsOp.region().op_begin())"
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 637, __extension__ __PRETTY_FUNCTION__))
;
638
639 // TODO: Perform appropriate actions according to the data-sharing
640 // attribute (shared, private, firstprivate, ...) of variables.
641 // Currently defaults to shared.
642 auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
643 llvm::Value &vPtr,
644 llvm::Value *&replacementValue) -> InsertPointTy {
645 replacementValue = &vPtr;
646 return codeGenIP;
647 };
648
649 // TODO: Perform finalization actions for variables. This has to be
650 // called for variables which have destructors/finalizers.
651 auto finiCB = [&](InsertPointTy codeGenIP) {};
652
653 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
654 findAllocaInsertPoint(builder, moduleTranslation);
655 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
656 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSections(
657 ompLoc, allocaIP, sectionCBs, privCB, finiCB, false,
658 sectionsOp.nowait()));
659 return bodyGenStatus;
660}
661
662/// Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.
663static LogicalResult
664convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder,
665 LLVM::ModuleTranslation &moduleTranslation) {
666 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
667 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
668 LogicalResult bodyGenStatus = success();
669 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
670 builder.restoreIP(codegenIP);
671 convertOmpOpRegions(singleOp.region(), "omp.single.region", builder,
672 moduleTranslation, bodyGenStatus);
673 };
674 auto finiCB = [&](InsertPointTy codeGenIP) {};
675 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createSingle(
676 ompLoc, bodyCB, finiCB, singleOp.nowait(), /*DidIt=*/nullptr));
677 return bodyGenStatus;
678}
679
680/// Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.
681static LogicalResult
682convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
683 LLVM::ModuleTranslation &moduleTranslation) {
684 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
685 LogicalResult bodyGenStatus = success();
686 if (taskOp.if_expr() || taskOp.final_expr() || taskOp.untiedAttr() ||
687 taskOp.mergeableAttr() || taskOp.in_reductions() || taskOp.priority() ||
688 !taskOp.allocate_vars().empty()) {
689 return taskOp.emitError("unhandled clauses for translation to LLVM IR");
690 }
691 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
692 builder.restoreIP(codegenIP);
693 convertOmpOpRegions(taskOp.region(), "omp.task.region", builder,
694 moduleTranslation, bodyGenStatus);
695 };
696 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
697 findAllocaInsertPoint(builder, moduleTranslation);
698 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
699 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTask(
700 ompLoc, allocaIP, bodyCB, !taskOp.untied()));
701 return bodyGenStatus;
702}
703
704/// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.
705static LogicalResult
706convertOmpTaskgroupOp(omp::TaskGroupOp tgOp, llvm::IRBuilderBase &builder,
707 LLVM::ModuleTranslation &moduleTranslation) {
708 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
709 LogicalResult bodyGenStatus = success();
710 if (!tgOp.task_reduction_vars().empty() || !tgOp.allocate_vars().empty()) {
711 return tgOp.emitError("unhandled clauses for translation to LLVM IR");
712 }
713 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP) {
714 builder.restoreIP(codegenIP);
715 convertOmpOpRegions(tgOp.region(), "omp.taskgroup.region", builder,
716 moduleTranslation, bodyGenStatus);
717 };
718 InsertPointTy allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
719 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
720 builder.restoreIP(moduleTranslation.getOpenMPBuilder()->createTaskgroup(
721 ompLoc, allocaIP, bodyCB));
722 return bodyGenStatus;
723}
724
725/// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
726static LogicalResult
727convertOmpWsLoop(Operation &opInst, llvm::IRBuilderBase &builder,
728 LLVM::ModuleTranslation &moduleTranslation) {
729 auto loop = cast<omp::WsLoopOp>(opInst);
730 // TODO: this should be in the op verifier instead.
731 if (loop.lowerBound().empty())
732 return failure();
733
734 // Static is the default.
735 auto schedule = loop.schedule_val().value_or(omp::ClauseScheduleKind::Static);
736
737 // Find the loop configuration.
738 llvm::Value *step = moduleTranslation.lookupValue(loop.step()[0]);
739 llvm::Type *ivType = step->getType();
740 llvm::Value *chunk = nullptr;
741 if (loop.schedule_chunk_var()) {
742 llvm::Value *chunkVar =
743 moduleTranslation.lookupValue(loop.schedule_chunk_var());
744 chunk = builder.CreateSExtOrTrunc(chunkVar, ivType);
745 }
746
747 SmallVector<omp::ReductionDeclareOp> reductionDecls;
748 collectReductionDecls(loop, reductionDecls);
749 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
750 findAllocaInsertPoint(builder, moduleTranslation);
751
752 // Allocate space for privatized reduction variables.
753 SmallVector<llvm::Value *> privateReductionVariables;
754 DenseMap<Value, llvm::Value *> reductionVariableMap;
755 unsigned numReductions = loop.getNumReductionVars();
756 privateReductionVariables.reserve(numReductions);
757 if (numReductions != 0) {
758 llvm::IRBuilderBase::InsertPointGuard guard(builder);
759 builder.restoreIP(allocaIP);
760 for (unsigned i = 0; i < numReductions; ++i) {
761 auto reductionType =
762 loop.reduction_vars()[i].getType().cast<LLVM::LLVMPointerType>();
763 llvm::Value *var = builder.CreateAlloca(
764 moduleTranslation.convertType(reductionType.getElementType()));
765 privateReductionVariables.push_back(var);
766 reductionVariableMap.try_emplace(loop.reduction_vars()[i], var);
767 }
768 }
769
770 // Store the mapping between reduction variables and their private copies on
771 // ModuleTranslation stack. It can be then recovered when translating
772 // omp.reduce operations in a separate call.
773 LLVM::ModuleTranslation::SaveStack<OpenMPVarMappingStackFrame> mappingGuard(
774 moduleTranslation, reductionVariableMap);
775
776 // Before the loop, store the initial values of reductions into reduction
777 // variables. Although this could be done after allocas, we don't want to mess
778 // up with the alloca insertion point.
779 for (unsigned i = 0; i < numReductions; ++i) {
780 SmallVector<llvm::Value *> phis;
781 if (failed(inlineConvertOmpRegions(reductionDecls[i].initializerRegion(),
782 "omp.reduction.neutral", builder,
783 moduleTranslation, &phis)))
784 return failure();
785 assert(phis.size() == 1 && "expected one value to be yielded from the "(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from the "
"reduction neutral element declaration region") ? void (0) :
__assert_fail ("phis.size() == 1 && \"expected one value to be yielded from the \" \"reduction neutral element declaration region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 786, __extension__ __PRETTY_FUNCTION__))
786 "reduction neutral element declaration region")(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from the "
"reduction neutral element declaration region") ? void (0) :
__assert_fail ("phis.size() == 1 && \"expected one value to be yielded from the \" \"reduction neutral element declaration region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 786, __extension__ __PRETTY_FUNCTION__))
;
787 builder.CreateStore(phis[0], privateReductionVariables[i]);
788 }
789
790 // Set up the source location value for OpenMP runtime.
791 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
792
793 // Generator of the canonical loop body.
794 // TODO: support error propagation in OpenMPIRBuilder and use it instead of
795 // relying on captured variables.
796 SmallVector<llvm::CanonicalLoopInfo *> loopInfos;
797 SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;
798 LogicalResult bodyGenStatus = success();
799 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
800 // Make sure further conversions know about the induction variable.
801 moduleTranslation.mapValue(
802 loop.getRegion().front().getArgument(loopInfos.size()), iv);
803
804 // Capture the body insertion point for use in nested loops. BodyIP of the
805 // CanonicalLoopInfo always points to the beginning of the entry block of
806 // the body.
807 bodyInsertPoints.push_back(ip);
808
809 if (loopInfos.size() != loop.getNumLoops() - 1)
810 return;
811
812 // Convert the body of the loop.
813 builder.restoreIP(ip);
814 convertOmpOpRegions(loop.region(), "omp.wsloop.region", builder,
815 moduleTranslation, bodyGenStatus);
816 };
817
818 // Delegate actual loop construction to the OpenMP IRBuilder.
819 // TODO: this currently assumes WsLoop is semantically similar to SCF loop,
820 // i.e. it has a positive step, uses signed integer semantics. Reconsider
821 // this code when WsLoop clearly supports more cases.
822 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
823 for (unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) {
824 llvm::Value *lowerBound =
825 moduleTranslation.lookupValue(loop.lowerBound()[i]);
826 llvm::Value *upperBound =
827 moduleTranslation.lookupValue(loop.upperBound()[i]);
828 llvm::Value *step = moduleTranslation.lookupValue(loop.step()[i]);
829
830 // Make sure loop trip count are emitted in the preheader of the outermost
831 // loop at the latest so that they are all available for the new collapsed
832 // loop will be created below.
833 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
834 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
835 if (i != 0) {
836 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back());
837 computeIP = loopInfos.front()->getPreheaderIP();
838 }
839 loopInfos.push_back(ompBuilder->createCanonicalLoop(
840 loc, bodyGen, lowerBound, upperBound, step,
841 /*IsSigned=*/true, loop.inclusive(), computeIP));
842
843 if (failed(bodyGenStatus))
844 return failure();
845 }
846
847 // Collapse loops. Store the insertion point because LoopInfos may get
848 // invalidated.
849 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
850 llvm::CanonicalLoopInfo *loopInfo =
851 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
852
853 allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
854
855 // TODO: Handle doacross loops when the ordered clause has a parameter.
856 bool isOrdered = loop.ordered_val().has_value();
857 Optional<omp::ScheduleModifier> scheduleModifier = loop.schedule_modifier();
858 bool isSimd = loop.simd_modifier();
859
860 ompBuilder->applyWorkshareLoop(
861 ompLoc.DL, loopInfo, allocaIP, !loop.nowait(),
862 convertToScheduleKind(schedule), chunk, isSimd,
863 scheduleModifier == omp::ScheduleModifier::monotonic,
864 scheduleModifier == omp::ScheduleModifier::nonmonotonic, isOrdered);
865
866 // Continue building IR after the loop. Note that the LoopInfo returned by
867 // `collapseLoops` points inside the outermost loop and is intended for
868 // potential further loop transformations. Use the insertion point stored
869 // before collapsing loops instead.
870 builder.restoreIP(afterIP);
871
872 // Process the reductions if required.
873 if (numReductions == 0)
874 return success();
875
876 // Create the reduction generators. We need to own them here because
877 // ReductionInfo only accepts references to the generators.
878 SmallVector<OwningReductionGen> owningReductionGens;
879 SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
880 for (unsigned i = 0; i < numReductions; ++i) {
881 owningReductionGens.push_back(
882 makeReductionGen(reductionDecls[i], builder, moduleTranslation));
883 owningAtomicReductionGens.push_back(
884 makeAtomicReductionGen(reductionDecls[i], builder, moduleTranslation));
885 }
886
887 // Collect the reduction information.
888 SmallVector<llvm::OpenMPIRBuilder::ReductionInfo> reductionInfos;
889 reductionInfos.reserve(numReductions);
890 for (unsigned i = 0; i < numReductions; ++i) {
891 llvm::OpenMPIRBuilder::AtomicReductionGenTy atomicGen = nullptr;
892 if (owningAtomicReductionGens[i])
893 atomicGen = owningAtomicReductionGens[i];
894 auto reductionType =
895 loop.reduction_vars()[i].getType().cast<LLVM::LLVMPointerType>();
896 llvm::Value *variable =
897 moduleTranslation.lookupValue(loop.reduction_vars()[i]);
898 reductionInfos.push_back(
899 {moduleTranslation.convertType(reductionType.getElementType()),
900 variable, privateReductionVariables[i], owningReductionGens[i],
901 atomicGen});
902 }
903
904 // The call to createReductions below expects the block to have a
905 // terminator. Create an unreachable instruction to serve as terminator
906 // and remove it later.
907 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
908 builder.SetInsertPoint(tempTerminator);
909 llvm::OpenMPIRBuilder::InsertPointTy contInsertPoint =
910 ompBuilder->createReductions(builder.saveIP(), allocaIP, reductionInfos,
911 loop.nowait());
912 if (!contInsertPoint.getBlock())
913 return loop->emitOpError() << "failed to convert reductions";
914 auto nextInsertionPoint =
915 ompBuilder->createBarrier(contInsertPoint, llvm::omp::OMPD_for);
916 tempTerminator->eraseFromParent();
917 builder.restoreIP(nextInsertionPoint);
918
919 return success();
920}
921
922/// Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.
923static LogicalResult
924convertOmpSimdLoop(Operation &opInst, llvm::IRBuilderBase &builder,
925 LLVM::ModuleTranslation &moduleTranslation) {
926 auto loop = cast<omp::SimdLoopOp>(opInst);
927
928 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
929
930 // Generator of the canonical loop body.
931 // TODO: support error propagation in OpenMPIRBuilder and use it instead of
932 // relying on captured variables.
933 SmallVector<llvm::CanonicalLoopInfo *> loopInfos;
934 SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;
935 LogicalResult bodyGenStatus = success();
936 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *iv) {
937 // Make sure further conversions know about the induction variable.
938 moduleTranslation.mapValue(
939 loop.getRegion().front().getArgument(loopInfos.size()), iv);
940
941 // Capture the body insertion point for use in nested loops. BodyIP of the
942 // CanonicalLoopInfo always points to the beginning of the entry block of
943 // the body.
944 bodyInsertPoints.push_back(ip);
945
946 if (loopInfos.size() != loop.getNumLoops() - 1)
1
Assuming the condition is false
2
Taking false branch
947 return;
948
949 // Convert the body of the loop.
950 builder.restoreIP(ip);
951 convertOmpOpRegions(loop.region(), "omp.simdloop.region", builder,
3
Passing null pointer value via 6th parameter 'continuationBlockPHIs'
4
Calling 'convertOmpOpRegions'
952 moduleTranslation, bodyGenStatus);
953 };
954
955 // Delegate actual loop construction to the OpenMP IRBuilder.
956 // TODO: this currently assumes SimdLoop is semantically similar to SCF loop,
957 // i.e. it has a positive step, uses signed integer semantics. Reconsider
958 // this code when SimdLoop clearly supports more cases.
959 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
960 for (unsigned i = 0, e = loop.getNumLoops(); i < e; ++i) {
961 llvm::Value *lowerBound =
962 moduleTranslation.lookupValue(loop.lowerBound()[i]);
963 llvm::Value *upperBound =
964 moduleTranslation.lookupValue(loop.upperBound()[i]);
965 llvm::Value *step = moduleTranslation.lookupValue(loop.step()[i]);
966
967 // Make sure loop trip count are emitted in the preheader of the outermost
968 // loop at the latest so that they are all available for the new collapsed
969 // loop will be created below.
970 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
971 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
972 if (i != 0) {
973 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),
974 ompLoc.DL);
975 computeIP = loopInfos.front()->getPreheaderIP();
976 }
977 loopInfos.push_back(ompBuilder->createCanonicalLoop(
978 loc, bodyGen, lowerBound, upperBound, step,
979 /*IsSigned=*/true, /*Inclusive=*/true, computeIP));
980
981 if (failed(bodyGenStatus))
982 return failure();
983 }
984
985 // Collapse loops.
986 llvm::IRBuilderBase::InsertPoint afterIP = loopInfos.front()->getAfterIP();
987 llvm::CanonicalLoopInfo *loopInfo =
988 ompBuilder->collapseLoops(ompLoc.DL, loopInfos, {});
989
990 llvm::ConstantInt *simdlen = nullptr;
991 if (llvm::Optional<uint64_t> simdlenVar = loop.simdlen())
992 simdlen = builder.getInt64(simdlenVar.value());
993
994 llvm::ConstantInt *safelen = nullptr;
995 if (llvm::Optional<uint64_t> safelenVar = loop.safelen())
996 safelen = builder.getInt64(safelenVar.value());
997
998 ompBuilder->applySimd(
999 loopInfo,
1000 loop.if_expr() ? moduleTranslation.lookupValue(loop.if_expr()) : nullptr,
1001 simdlen, safelen);
1002
1003 builder.restoreIP(afterIP);
1004 return success();
1005}
1006
1007/// Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
1008llvm::AtomicOrdering
1009convertAtomicOrdering(Optional<omp::ClauseMemoryOrderKind> ao) {
1010 if (!ao)
1011 return llvm::AtomicOrdering::Monotonic; // Default Memory Ordering
1012
1013 switch (*ao) {
1014 case omp::ClauseMemoryOrderKind::Seq_cst:
1015 return llvm::AtomicOrdering::SequentiallyConsistent;
1016 case omp::ClauseMemoryOrderKind::Acq_rel:
1017 return llvm::AtomicOrdering::AcquireRelease;
1018 case omp::ClauseMemoryOrderKind::Acquire:
1019 return llvm::AtomicOrdering::Acquire;
1020 case omp::ClauseMemoryOrderKind::Release:
1021 return llvm::AtomicOrdering::Release;
1022 case omp::ClauseMemoryOrderKind::Relaxed:
1023 return llvm::AtomicOrdering::Monotonic;
1024 }
1025 llvm_unreachable("Unknown ClauseMemoryOrderKind kind")::llvm::llvm_unreachable_internal("Unknown ClauseMemoryOrderKind kind"
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1025)
;
1026}
1027
1028/// Convert omp.atomic.read operation to LLVM IR.
1029static LogicalResult
1030convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder,
1031 LLVM::ModuleTranslation &moduleTranslation) {
1032
1033 auto readOp = cast<omp::AtomicReadOp>(opInst);
1034 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
1035
1036 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1037
1038 llvm::AtomicOrdering AO = convertAtomicOrdering(readOp.memory_order_val());
1039 llvm::Value *x = moduleTranslation.lookupValue(readOp.x());
1040 Type xTy = readOp.x().getType().cast<omp::PointerLikeType>().getElementType();
1041 llvm::Value *v = moduleTranslation.lookupValue(readOp.v());
1042 Type vTy = readOp.v().getType().cast<omp::PointerLikeType>().getElementType();
1043 llvm::OpenMPIRBuilder::AtomicOpValue V = {
1044 v, moduleTranslation.convertType(vTy), false, false};
1045 llvm::OpenMPIRBuilder::AtomicOpValue X = {
1046 x, moduleTranslation.convertType(xTy), false, false};
1047 builder.restoreIP(ompBuilder->createAtomicRead(ompLoc, X, V, AO));
1048 return success();
1049}
1050
1051/// Converts an omp.atomic.write operation to LLVM IR.
1052static LogicalResult
1053convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder,
1054 LLVM::ModuleTranslation &moduleTranslation) {
1055 auto writeOp = cast<omp::AtomicWriteOp>(opInst);
1056 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
1057
1058 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1059 llvm::AtomicOrdering ao = convertAtomicOrdering(writeOp.memory_order_val());
1060 llvm::Value *expr = moduleTranslation.lookupValue(writeOp.value());
1061 llvm::Value *dest = moduleTranslation.lookupValue(writeOp.address());
1062 llvm::Type *ty = moduleTranslation.convertType(writeOp.value().getType());
1063 llvm::OpenMPIRBuilder::AtomicOpValue x = {dest, ty, /*isSigned=*/false,
1064 /*isVolatile=*/false};
1065 builder.restoreIP(ompBuilder->createAtomicWrite(ompLoc, x, expr, ao));
1066 return success();
1067}
1068
1069/// Converts an LLVM dialect binary operation to the corresponding enum value
1070/// for `atomicrmw` supported binary operation.
1071llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) {
1072 return llvm::TypeSwitch<Operation *, llvm::AtomicRMWInst::BinOp>(&op)
1073 .Case([&](LLVM::AddOp) { return llvm::AtomicRMWInst::BinOp::Add; })
1074 .Case([&](LLVM::SubOp) { return llvm::AtomicRMWInst::BinOp::Sub; })
1075 .Case([&](LLVM::AndOp) { return llvm::AtomicRMWInst::BinOp::And; })
1076 .Case([&](LLVM::OrOp) { return llvm::AtomicRMWInst::BinOp::Or; })
1077 .Case([&](LLVM::XOrOp) { return llvm::AtomicRMWInst::BinOp::Xor; })
1078 .Case([&](LLVM::UMaxOp) { return llvm::AtomicRMWInst::BinOp::UMax; })
1079 .Case([&](LLVM::UMinOp) { return llvm::AtomicRMWInst::BinOp::UMin; })
1080 .Case([&](LLVM::FAddOp) { return llvm::AtomicRMWInst::BinOp::FAdd; })
1081 .Case([&](LLVM::FSubOp) { return llvm::AtomicRMWInst::BinOp::FSub; })
1082 .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
1083}
1084
1085/// Converts an OpenMP atomic update operation using OpenMPIRBuilder.
1086static LogicalResult
1087convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
1088 llvm::IRBuilderBase &builder,
1089 LLVM::ModuleTranslation &moduleTranslation) {
1090 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
1091
1092 // Convert values and types.
1093 auto &innerOpList = opInst.region().front().getOperations();
1094 if (innerOpList.size() != 2)
1095 return opInst.emitError("exactly two operations are allowed inside an "
1096 "atomic update region while lowering to LLVM IR");
1097
1098 Operation &innerUpdateOp = innerOpList.front();
1099
1100 if (innerUpdateOp.getNumOperands() != 2 ||
1101 !llvm::is_contained(innerUpdateOp.getOperands(),
1102 opInst.getRegion().getArgument(0)))
1103 return opInst.emitError(
1104 "the update operation inside the region must be a binary operation and "
1105 "that update operation must have the region argument as an operand");
1106
1107 llvm::AtomicRMWInst::BinOp binop = convertBinOpToAtomic(innerUpdateOp);
1108
1109 bool isXBinopExpr =
1110 innerUpdateOp.getNumOperands() > 0 &&
1111 innerUpdateOp.getOperand(0) == opInst.getRegion().getArgument(0);
1112
1113 mlir::Value mlirExpr = (isXBinopExpr ? innerUpdateOp.getOperand(1)
1114 : innerUpdateOp.getOperand(0));
1115 llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr);
1116 llvm::Value *llvmX = moduleTranslation.lookupValue(opInst.x());
1117 LLVM::LLVMPointerType mlirXType =
1118 opInst.x().getType().cast<LLVM::LLVMPointerType>();
1119 llvm::Type *llvmXElementType =
1120 moduleTranslation.convertType(mlirXType.getElementType());
1121 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1122 /*isSigned=*/false,
1123 /*isVolatile=*/false};
1124
1125 llvm::AtomicOrdering atomicOrdering =
1126 convertAtomicOrdering(opInst.memory_order_val());
1127
1128 // Generate update code.
1129 LogicalResult updateGenStatus = success();
1130 auto updateFn = [&opInst, &moduleTranslation, &updateGenStatus](
1131 llvm::Value *atomicx,
1132 llvm::IRBuilder<> &builder) -> llvm::Value * {
1133 Block &bb = *opInst.region().begin();
1134 moduleTranslation.mapValue(*opInst.region().args_begin(), atomicx);
1135 moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());
1136 if (failed(moduleTranslation.convertBlock(bb, true, builder))) {
1137 updateGenStatus = (opInst.emitError()
1138 << "unable to convert update operation to llvm IR");
1139 return nullptr;
1140 }
1141 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator());
1142 assert(yieldop && yieldop.results().size() == 1 &&(static_cast <bool> (yieldop && yieldop.results
().size() == 1 && "terminator must be omp.yield op and it must have exactly one "
"argument") ? void (0) : __assert_fail ("yieldop && yieldop.results().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1144, __extension__ __PRETTY_FUNCTION__))
1143 "terminator must be omp.yield op and it must have exactly one "(static_cast <bool> (yieldop && yieldop.results
().size() == 1 && "terminator must be omp.yield op and it must have exactly one "
"argument") ? void (0) : __assert_fail ("yieldop && yieldop.results().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1144, __extension__ __PRETTY_FUNCTION__))
1144 "argument")(static_cast <bool> (yieldop && yieldop.results
().size() == 1 && "terminator must be omp.yield op and it must have exactly one "
"argument") ? void (0) : __assert_fail ("yieldop && yieldop.results().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1144, __extension__ __PRETTY_FUNCTION__))
;
1145 return moduleTranslation.lookupValue(yieldop.results()[0]);
1146 };
1147
1148 // Handle ambiguous alloca, if any.
1149 auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
1150 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1151 builder.restoreIP(ompBuilder->createAtomicUpdate(
1152 ompLoc, allocaIP, llvmAtomicX, llvmExpr, atomicOrdering, binop, updateFn,
1153 isXBinopExpr));
1154 return updateGenStatus;
1155}
1156
1157static LogicalResult
1158convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
1159 llvm::IRBuilderBase &builder,
1160 LLVM::ModuleTranslation &moduleTranslation) {
1161 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
1162 mlir::Value mlirExpr;
1163 bool isXBinopExpr = false, isPostfixUpdate = false;
1164 llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
1165
1166 omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
1167 omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
1168
1169 assert((atomicUpdateOp || atomicWriteOp) &&(static_cast <bool> ((atomicUpdateOp || atomicWriteOp) &&
"internal op must be an atomic.update or atomic.write op") ?
void (0) : __assert_fail ("(atomicUpdateOp || atomicWriteOp) && \"internal op must be an atomic.update or atomic.write op\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1170, __extension__ __PRETTY_FUNCTION__))
1170 "internal op must be an atomic.update or atomic.write op")(static_cast <bool> ((atomicUpdateOp || atomicWriteOp) &&
"internal op must be an atomic.update or atomic.write op") ?
void (0) : __assert_fail ("(atomicUpdateOp || atomicWriteOp) && \"internal op must be an atomic.update or atomic.write op\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1170, __extension__ __PRETTY_FUNCTION__))
;
1171
1172 if (atomicWriteOp) {
1173 isPostfixUpdate = true;
1174 mlirExpr = atomicWriteOp.value();
1175 } else {
1176 isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
1177 atomicCaptureOp.getAtomicUpdateOp().getOperation();
1178 auto &innerOpList = atomicUpdateOp.region().front().getOperations();
1179 if (innerOpList.size() != 2)
1180 return atomicUpdateOp.emitError(
1181 "exactly two operations are allowed inside an "
1182 "atomic update region while lowering to LLVM IR");
1183 Operation *innerUpdateOp = atomicUpdateOp.getFirstOp();
1184 if (innerUpdateOp->getNumOperands() != 2 ||
1185 !llvm::is_contained(innerUpdateOp->getOperands(),
1186 atomicUpdateOp.getRegion().getArgument(0)))
1187 return atomicUpdateOp.emitError(
1188 "the update operation inside the region must be a binary operation "
1189 "and that update operation must have the region argument as an "
1190 "operand");
1191 binop = convertBinOpToAtomic(*innerUpdateOp);
1192
1193 isXBinopExpr = innerUpdateOp->getOperand(0) ==
1194 atomicUpdateOp.getRegion().getArgument(0);
1195
1196 mlirExpr = (isXBinopExpr ? innerUpdateOp->getOperand(1)
1197 : innerUpdateOp->getOperand(0));
1198 }
1199
1200 llvm::Value *llvmExpr = moduleTranslation.lookupValue(mlirExpr);
1201 llvm::Value *llvmX =
1202 moduleTranslation.lookupValue(atomicCaptureOp.getAtomicReadOp().x());
1203 llvm::Value *llvmV =
1204 moduleTranslation.lookupValue(atomicCaptureOp.getAtomicReadOp().v());
1205 auto mlirXType = atomicCaptureOp.getAtomicReadOp()
1206 .x()
1207 .getType()
1208 .cast<LLVM::LLVMPointerType>();
1209 llvm::Type *llvmXElementType =
1210 moduleTranslation.convertType(mlirXType.getElementType());
1211 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {llvmX, llvmXElementType,
1212 /*isSigned=*/false,
1213 /*isVolatile=*/false};
1214 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {llvmV, llvmXElementType,
1215 /*isSigned=*/false,
1216 /*isVolatile=*/false};
1217
1218 llvm::AtomicOrdering atomicOrdering =
1219 convertAtomicOrdering(atomicCaptureOp.memory_order_val());
1220
1221 LogicalResult updateGenStatus = success();
1222 auto updateFn = [&](llvm::Value *atomicx,
1223 llvm::IRBuilder<> &builder) -> llvm::Value * {
1224 if (atomicWriteOp)
1225 return moduleTranslation.lookupValue(atomicWriteOp.value());
1226 Block &bb = *atomicUpdateOp.region().begin();
1227 moduleTranslation.mapValue(*atomicUpdateOp.region().args_begin(), atomicx);
1228 moduleTranslation.mapBlock(&bb, builder.GetInsertBlock());
1229 if (failed(moduleTranslation.convertBlock(bb, true, builder))) {
1230 updateGenStatus = (atomicUpdateOp.emitError()
1231 << "unable to convert update operation to llvm IR");
1232 return nullptr;
1233 }
1234 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(bb.getTerminator());
1235 assert(yieldop && yieldop.results().size() == 1 &&(static_cast <bool> (yieldop && yieldop.results
().size() == 1 && "terminator must be omp.yield op and it must have exactly one "
"argument") ? void (0) : __assert_fail ("yieldop && yieldop.results().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1237, __extension__ __PRETTY_FUNCTION__))
1236 "terminator must be omp.yield op and it must have exactly one "(static_cast <bool> (yieldop && yieldop.results
().size() == 1 && "terminator must be omp.yield op and it must have exactly one "
"argument") ? void (0) : __assert_fail ("yieldop && yieldop.results().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1237, __extension__ __PRETTY_FUNCTION__))
1237 "argument")(static_cast <bool> (yieldop && yieldop.results
().size() == 1 && "terminator must be omp.yield op and it must have exactly one "
"argument") ? void (0) : __assert_fail ("yieldop && yieldop.results().size() == 1 && \"terminator must be omp.yield op and it must have exactly one \" \"argument\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1237, __extension__ __PRETTY_FUNCTION__))
;
1238 return moduleTranslation.lookupValue(yieldop.results()[0]);
1239 };
1240
1241 // Handle ambiguous alloca, if any.
1242 auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
1243 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1244 builder.restoreIP(ompBuilder->createAtomicCapture(
1245 ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
1246 binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr));
1247 return updateGenStatus;
1248}
1249
1250/// Converts an OpenMP reduction operation using OpenMPIRBuilder. Expects the
1251/// mapping between reduction variables and their private equivalents to have
1252/// been stored on the ModuleTranslation stack. Currently only supports
1253/// reduction within WsLoopOp, but can be easily extended.
1254static LogicalResult
1255convertOmpReductionOp(omp::ReductionOp reductionOp,
1256 llvm::IRBuilderBase &builder,
1257 LLVM::ModuleTranslation &moduleTranslation) {
1258 // Find the declaration that corresponds to the reduction op.
1259 auto reductionContainer = reductionOp->getParentOfType<omp::WsLoopOp>();
1260 omp::ReductionDeclareOp declaration =
1261 findReductionDecl(reductionContainer, reductionOp);
1262 assert(declaration && "could not find reduction declaration")(static_cast <bool> (declaration && "could not find reduction declaration"
) ? void (0) : __assert_fail ("declaration && \"could not find reduction declaration\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1262, __extension__ __PRETTY_FUNCTION__))
;
1263
1264 // Retrieve the mapping between reduction variables and their private
1265 // equivalents.
1266 const DenseMap<Value, llvm::Value *> *reductionVariableMap = nullptr;
1267 moduleTranslation.stackWalk<OpenMPVarMappingStackFrame>(
1268 [&](const OpenMPVarMappingStackFrame &frame) {
1269 reductionVariableMap = &frame.mapping;
1270 return WalkResult::interrupt();
1271 });
1272 assert(reductionVariableMap && "couldn't find private reduction variables")(static_cast <bool> (reductionVariableMap && "couldn't find private reduction variables"
) ? void (0) : __assert_fail ("reductionVariableMap && \"couldn't find private reduction variables\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1272, __extension__ __PRETTY_FUNCTION__))
;
1273
1274 // Translate the reduction operation by emitting the body of the corresponding
1275 // reduction declaration.
1276 Region &reductionRegion = declaration.reductionRegion();
1277 llvm::Value *privateReductionVar =
1278 reductionVariableMap->lookup(reductionOp.accumulator());
1279 llvm::Value *reductionVal = builder.CreateLoad(
1280 moduleTranslation.convertType(reductionOp.operand().getType()),
1281 privateReductionVar);
1282
1283 moduleTranslation.mapValue(reductionRegion.front().getArgument(0),
1284 reductionVal);
1285 moduleTranslation.mapValue(
1286 reductionRegion.front().getArgument(1),
1287 moduleTranslation.lookupValue(reductionOp.operand()));
1288
1289 SmallVector<llvm::Value *> phis;
1290 if (failed(inlineConvertOmpRegions(reductionRegion, "omp.reduction.body",
1291 builder, moduleTranslation, &phis)))
1292 return failure();
1293 assert(phis.size() == 1 && "expected one value to be yielded from "(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from "
"the reduction body declaration region") ? void (0) : __assert_fail
("phis.size() == 1 && \"expected one value to be yielded from \" \"the reduction body declaration region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1294, __extension__ __PRETTY_FUNCTION__))
1294 "the reduction body declaration region")(static_cast <bool> (phis.size() == 1 && "expected one value to be yielded from "
"the reduction body declaration region") ? void (0) : __assert_fail
("phis.size() == 1 && \"expected one value to be yielded from \" \"the reduction body declaration region\""
, "mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp"
, 1294, __extension__ __PRETTY_FUNCTION__))
;
1295 builder.CreateStore(phis[0], privateReductionVar);
1296 return success();
1297}
1298
1299/// Converts an OpenMP Threadprivate operation into LLVM IR using
1300/// OpenMPIRBuilder.
1301static LogicalResult
1302convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder,
1303 LLVM::ModuleTranslation &moduleTranslation) {
1304 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1305 auto threadprivateOp = cast<omp::ThreadprivateOp>(opInst);
1306
1307 Value symAddr = threadprivateOp.sym_addr();
1308 auto *symOp = symAddr.getDefiningOp();
1309 if (!isa<LLVM::AddressOfOp>(symOp))
1310 return opInst.emitError("Addressing symbol not found");
1311 LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(symOp);
1312
1313 LLVM::GlobalOp global =
1314 addressOfOp.getGlobal(moduleTranslation.symbolTable());
1315 llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(global);
1316 llvm::Value *data =
1317 builder.CreateBitCast(globalValue, builder.getInt8PtrTy());
1318 llvm::Type *type = globalValue->getValueType();
1319 llvm::TypeSize typeSize =
1320 builder.GetInsertBlock()->getModule()->getDataLayout().getTypeStoreSize(
1321 type);
1322 llvm::ConstantInt *size = builder.getInt64(typeSize.getFixedSize());
1323 llvm::StringRef suffix = llvm::StringRef(".cache", 6);
1324 std::string cacheName = (Twine(global.getSymName()).concat(suffix)).str();
1325 // Emit runtime function and bitcast its type (i8*) to real data type.
1326 llvm::Value *callInst =
1327 moduleTranslation.getOpenMPBuilder()->createCachedThreadPrivate(
1328 ompLoc, data, size, cacheName);
1329 llvm::Value *result = builder.CreateBitCast(callInst, globalValue->getType());
1330 moduleTranslation.mapValue(opInst.getResult(0), result);
1331 return success();
1332}
1333
1334namespace {
1335
1336/// Implementation of the dialect interface that converts operations belonging
1337/// to the OpenMP dialect to LLVM IR.
1338class OpenMPDialectLLVMIRTranslationInterface
1339 : public LLVMTranslationDialectInterface {
1340public:
1341 using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
1342
1343 /// Translates the given operation to LLVM IR using the provided IR builder
1344 /// and saving the state in `moduleTranslation`.
1345 LogicalResult
1346 convertOperation(Operation *op, llvm::IRBuilderBase &builder,
1347 LLVM::ModuleTranslation &moduleTranslation) const final;
1348};
1349
1350} // namespace
1351
1352/// Given an OpenMP MLIR operation, create the corresponding LLVM IR
1353/// (including OpenMP runtime calls).
1354LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
1355 Operation *op, llvm::IRBuilderBase &builder,
1356 LLVM::ModuleTranslation &moduleTranslation) const {
1357
1358 llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
1359
1360 return llvm::TypeSwitch<Operation *, LogicalResult>(op)
1361 .Case([&](omp::BarrierOp) {
1362 ompBuilder->createBarrier(builder.saveIP(), llvm::omp::OMPD_barrier);
1363 return success();
1364 })
1365 .Case([&](omp::TaskwaitOp) {
1366 ompBuilder->createTaskwait(builder.saveIP());
1367 return success();
1368 })
1369 .Case([&](omp::TaskyieldOp) {
1370 ompBuilder->createTaskyield(builder.saveIP());
1371 return success();
1372 })
1373 .Case([&](omp::FlushOp) {
1374 // No support in Openmp runtime function (__kmpc_flush) to accept
1375 // the argument list.
1376 // OpenMP standard states the following:
1377 // "An implementation may implement a flush with a list by ignoring
1378 // the list, and treating it the same as a flush without a list."
1379 //
1380 // The argument list is discarded so that, flush with a list is treated
1381 // same as a flush without a list.
1382 ompBuilder->createFlush(builder.saveIP());
1383 return success();
1384 })
1385 .Case([&](omp::ParallelOp op) {
1386 return convertOmpParallel(op, builder, moduleTranslation);
1387 })
1388 .Case([&](omp::ReductionOp reductionOp) {
1389 return convertOmpReductionOp(reductionOp, builder, moduleTranslation);
1390 })
1391 .Case([&](omp::MasterOp) {
1392 return convertOmpMaster(*op, builder, moduleTranslation);
1393 })
1394 .Case([&](omp::CriticalOp) {
1395 return convertOmpCritical(*op, builder, moduleTranslation);
1396 })
1397 .Case([&](omp::OrderedRegionOp) {
1398 return convertOmpOrderedRegion(*op, builder, moduleTranslation);
1399 })
1400 .Case([&](omp::OrderedOp) {
1401 return convertOmpOrdered(*op, builder, moduleTranslation);
1402 })
1403 .Case([&](omp::WsLoopOp) {
1404 return convertOmpWsLoop(*op, builder, moduleTranslation);
1405 })
1406 .Case([&](omp::SimdLoopOp) {
1407 return convertOmpSimdLoop(*op, builder, moduleTranslation);
1408 })
1409 .Case([&](omp::AtomicReadOp) {
1410 return convertOmpAtomicRead(*op, builder, moduleTranslation);
1411 })
1412 .Case([&](omp::AtomicWriteOp) {
1413 return convertOmpAtomicWrite(*op, builder, moduleTranslation);
1414 })
1415 .Case([&](omp::AtomicUpdateOp op) {
1416 return convertOmpAtomicUpdate(op, builder, moduleTranslation);
1417 })
1418 .Case([&](omp::AtomicCaptureOp op) {
1419 return convertOmpAtomicCapture(op, builder, moduleTranslation);
1420 })
1421 .Case([&](omp::SectionsOp) {
1422 return convertOmpSections(*op, builder, moduleTranslation);
1423 })
1424 .Case([&](omp::SingleOp op) {
1425 return convertOmpSingle(op, builder, moduleTranslation);
1426 })
1427 .Case([&](omp::TaskOp op) {
1428 return convertOmpTaskOp(op, builder, moduleTranslation);
1429 })
1430 .Case([&](omp::TaskGroupOp op) {
1431 return convertOmpTaskgroupOp(op, builder, moduleTranslation);
1432 })
1433 .Case<omp::YieldOp, omp::TerminatorOp, omp::ReductionDeclareOp,
1434 omp::CriticalDeclareOp>([](auto op) {
1435 // `yield` and `terminator` can be just omitted. The block structure
1436 // was created in the region that handles their parent operation.
1437 // `reduction.declare` will be used by reductions and is not
1438 // converted directly, skip it.
1439 // `critical.declare` is only used to declare names of critical
1440 // sections which will be used by `critical` ops and hence can be
1441 // ignored for lowering. The OpenMP IRBuilder will create unique
1442 // name for critical section names.
1443 return success();
1444 })
1445 .Case([&](omp::ThreadprivateOp) {
1446 return convertOmpThreadprivate(*op, builder, moduleTranslation);
1447 })
1448 .Default([&](Operation *inst) {
1449 return inst->emitError("unsupported OpenMP operation: ")
1450 << inst->getName();
1451 });
1452}
1453
1454void mlir::registerOpenMPDialectTranslation(DialectRegistry &registry) {
1455 registry.insert<omp::OpenMPDialect>();
1456 registry.addExtension(+[](MLIRContext *ctx, omp::OpenMPDialect *dialect) {
1457 dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();
1458 });
1459}
1460
1461void mlir::registerOpenMPDialectTranslation(MLIRContext &context) {
1462 DialectRegistry registry;
1463 registerOpenMPDialectTranslation(registry);
1464 context.appendDialectRegistry(registry);
1465}