File: | build/source/openmp/libomptarget/src/api.cpp |
Warning: | line 368, column 3 Undefined or garbage value returned to caller |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===----------- api.cpp - Target independent OpenMP target RTL -----------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // Implementation of OpenMP API interface functions. | |||
10 | // | |||
11 | //===----------------------------------------------------------------------===// | |||
12 | ||||
13 | #include "device.h" | |||
14 | #include "omptarget.h" | |||
15 | #include "private.h" | |||
16 | #include "rtl.h" | |||
17 | ||||
18 | #include "llvm/ADT/SmallVector.h" | |||
19 | ||||
20 | #include <climits> | |||
21 | #include <cstdlib> | |||
22 | #include <cstring> | |||
23 | #include <mutex> | |||
24 | ||||
25 | EXTERNextern "C" int omp_get_num_devices(void) { | |||
26 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
27 | PM->RTLsMtx.lock(); | |||
28 | size_t DevicesSize = PM->Devices.size(); | |||
29 | PM->RTLsMtx.unlock(); | |||
30 | ||||
31 | DP("Call to omp_get_num_devices returning %zd\n", DevicesSize){}; | |||
32 | ||||
33 | return DevicesSize; | |||
34 | } | |||
35 | ||||
36 | EXTERNextern "C" int omp_get_device_num(void) { | |||
37 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
38 | int HostDevice = omp_get_initial_device(); | |||
39 | ||||
40 | DP("Call to omp_get_device_num returning %d\n", HostDevice){}; | |||
41 | ||||
42 | return HostDevice; | |||
43 | } | |||
44 | ||||
45 | EXTERNextern "C" int omp_get_initial_device(void) { | |||
46 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
47 | int HostDevice = omp_get_num_devices(); | |||
48 | DP("Call to omp_get_initial_device returning %d\n", HostDevice){}; | |||
49 | return HostDevice; | |||
50 | } | |||
51 | ||||
52 | EXTERNextern "C" void *omp_target_alloc(size_t Size, int DeviceNum) { | |||
53 | return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEFAULT, __func__); | |||
54 | } | |||
55 | ||||
56 | EXTERNextern "C" void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum) { | |||
57 | return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEVICE, __func__); | |||
58 | } | |||
59 | ||||
60 | EXTERNextern "C" void *llvm_omp_target_alloc_host(size_t Size, int DeviceNum) { | |||
61 | return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_HOST, __func__); | |||
62 | } | |||
63 | ||||
64 | EXTERNextern "C" void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum) { | |||
65 | return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_SHARED, __func__); | |||
66 | } | |||
67 | ||||
68 | EXTERNextern "C" void omp_target_free(void *Ptr, int DeviceNum) { | |||
69 | return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_DEFAULT, __func__); | |||
70 | } | |||
71 | ||||
72 | EXTERNextern "C" void llvm_omp_target_free_device(void *Ptr, int DeviceNum) { | |||
73 | return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_DEVICE, __func__); | |||
74 | } | |||
75 | ||||
76 | EXTERNextern "C" void llvm_omp_target_free_host(void *Ptr, int DeviceNum) { | |||
77 | return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_HOST, __func__); | |||
78 | } | |||
79 | ||||
80 | EXTERNextern "C" void llvm_omp_target_free_shared(void *Ptre, int DeviceNum) { | |||
81 | return targetFreeExplicit(Ptre, DeviceNum, TARGET_ALLOC_SHARED, __func__); | |||
82 | } | |||
83 | ||||
84 | EXTERNextern "C" void *llvm_omp_target_dynamic_shared_alloc() { return nullptr; } | |||
85 | EXTERNextern "C" void *llvm_omp_get_dynamic_shared() { return nullptr; } | |||
86 | ||||
87 | EXTERNextern "C" [[nodiscard]] void *llvm_omp_target_lock_mem(void *Ptr, size_t Size, | |||
88 | int DeviceNum) { | |||
89 | return targetLockExplicit(Ptr, Size, DeviceNum, __func__); | |||
90 | } | |||
91 | ||||
92 | EXTERNextern "C" void llvm_omp_target_unlock_mem(void *Ptr, int DeviceNum) { | |||
93 | targetUnlockExplicit(Ptr, DeviceNum, __func__); | |||
94 | } | |||
95 | ||||
96 | EXTERNextern "C" int omp_target_is_present(const void *Ptr, int DeviceNum) { | |||
97 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
98 | DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",{} | |||
99 | DeviceNum, DPxPTR(Ptr)){}; | |||
100 | ||||
101 | if (!Ptr) { | |||
102 | DP("Call to omp_target_is_present with NULL ptr, returning false\n"){}; | |||
103 | return false; | |||
104 | } | |||
105 | ||||
106 | if (DeviceNum == omp_get_initial_device()) { | |||
107 | DP("Call to omp_target_is_present on host, returning true\n"){}; | |||
108 | return true; | |||
109 | } | |||
110 | ||||
111 | PM->RTLsMtx.lock(); | |||
112 | size_t DevicesSize = PM->Devices.size(); | |||
113 | PM->RTLsMtx.unlock(); | |||
114 | if (DevicesSize <= (size_t)DeviceNum) { | |||
115 | DP("Call to omp_target_is_present with invalid device ID, returning "{} | |||
116 | "false\n"){}; | |||
117 | return false; | |||
118 | } | |||
119 | ||||
120 | DeviceTy &Device = *PM->Devices[DeviceNum]; | |||
121 | // omp_target_is_present tests whether a host pointer refers to storage that | |||
122 | // is mapped to a given device. However, due to the lack of the storage size, | |||
123 | // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero | |||
124 | // lengh array) is mapped instead of the referred storage. | |||
125 | TargetPointerResultTy TPR = Device.getTgtPtrBegin(const_cast<void *>(Ptr), 1, | |||
126 | /*UpdateRefCount=*/false, | |||
127 | /*UseHoldRefCount=*/false); | |||
128 | int Rc = TPR.isPresent(); | |||
129 | DP("Call to omp_target_is_present returns %d\n", Rc){}; | |||
130 | return Rc; | |||
131 | } | |||
132 | ||||
133 | EXTERNextern "C" int omp_target_memcpy(void *Dst, const void *Src, size_t Length, | |||
134 | size_t DstOffset, size_t SrcOffset, int DstDevice, | |||
135 | int SrcDevice) { | |||
136 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
137 | DP("Call to omp_target_memcpy, dst device %d, src device %d, "{} | |||
138 | "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "{} | |||
139 | "src offset %zu, length %zu\n",{} | |||
140 | DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,{} | |||
141 | Length){}; | |||
142 | ||||
143 | if (!Dst || !Src || Length <= 0) { | |||
144 | if (Length == 0) { | |||
145 | DP("Call to omp_target_memcpy with zero length, nothing to do\n"){}; | |||
146 | return OFFLOAD_SUCCESS(0); | |||
147 | } | |||
148 | ||||
149 | REPORT("Call to omp_target_memcpy with invalid arguments\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Call to omp_target_memcpy with invalid arguments\n"); } while (0);; | |||
150 | return OFFLOAD_FAIL(~0); | |||
151 | } | |||
152 | ||||
153 | if (SrcDevice != omp_get_initial_device() && !deviceIsReady(SrcDevice)) { | |||
154 | REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_memcpy returns OFFLOAD_FAIL\n"); } while (0);; | |||
155 | return OFFLOAD_FAIL(~0); | |||
156 | } | |||
157 | ||||
158 | if (DstDevice != omp_get_initial_device() && !deviceIsReady(DstDevice)) { | |||
159 | REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_memcpy returns OFFLOAD_FAIL\n"); } while (0);; | |||
160 | return OFFLOAD_FAIL(~0); | |||
161 | } | |||
162 | ||||
163 | int Rc = OFFLOAD_SUCCESS(0); | |||
164 | void *SrcAddr = (char *)const_cast<void *>(Src) + SrcOffset; | |||
165 | void *DstAddr = (char *)Dst + DstOffset; | |||
166 | ||||
167 | if (SrcDevice == omp_get_initial_device() && | |||
168 | DstDevice == omp_get_initial_device()) { | |||
169 | DP("copy from host to host\n"){}; | |||
170 | const void *P = memcpy(DstAddr, SrcAddr, Length); | |||
171 | if (P == NULL__null) | |||
172 | Rc = OFFLOAD_FAIL(~0); | |||
173 | } else if (SrcDevice == omp_get_initial_device()) { | |||
174 | DP("copy from host to device\n"){}; | |||
175 | DeviceTy &DstDev = *PM->Devices[DstDevice]; | |||
176 | AsyncInfoTy AsyncInfo(DstDev); | |||
177 | Rc = DstDev.submitData(DstAddr, SrcAddr, Length, AsyncInfo); | |||
178 | } else if (DstDevice == omp_get_initial_device()) { | |||
179 | DP("copy from device to host\n"){}; | |||
180 | DeviceTy &SrcDev = *PM->Devices[SrcDevice]; | |||
181 | AsyncInfoTy AsyncInfo(SrcDev); | |||
182 | Rc = SrcDev.retrieveData(DstAddr, SrcAddr, Length, AsyncInfo); | |||
183 | } else { | |||
184 | DP("copy from device to device\n"){}; | |||
185 | DeviceTy &SrcDev = *PM->Devices[SrcDevice]; | |||
186 | DeviceTy &DstDev = *PM->Devices[DstDevice]; | |||
187 | // First try to use D2D memcpy which is more efficient. If fails, fall back | |||
188 | // to unefficient way. | |||
189 | if (SrcDev.isDataExchangable(DstDev)) { | |||
190 | AsyncInfoTy AsyncInfo(SrcDev); | |||
191 | Rc = SrcDev.dataExchange(SrcAddr, DstDev, DstAddr, Length, AsyncInfo); | |||
192 | if (Rc == OFFLOAD_SUCCESS(0)) | |||
193 | return OFFLOAD_SUCCESS(0); | |||
194 | } | |||
195 | ||||
196 | void *Buffer = malloc(Length); | |||
197 | { | |||
198 | AsyncInfoTy AsyncInfo(SrcDev); | |||
199 | Rc = SrcDev.retrieveData(Buffer, SrcAddr, Length, AsyncInfo); | |||
200 | } | |||
201 | if (Rc == OFFLOAD_SUCCESS(0)) { | |||
202 | AsyncInfoTy AsyncInfo(DstDev); | |||
203 | Rc = DstDev.submitData(DstAddr, Buffer, Length, AsyncInfo); | |||
204 | } | |||
205 | free(Buffer); | |||
206 | } | |||
207 | ||||
208 | DP("omp_target_memcpy returns %d\n", Rc){}; | |||
209 | return Rc; | |||
210 | } | |||
211 | ||||
212 | // The helper function that calls omp_target_memcpy or omp_target_memcpy_rect | |||
213 | static int libomp_target_memcpy_async_helper(kmp_int32 Gtid, kmp_task_t *Task) { | |||
214 | if (Task == nullptr) | |||
| ||||
215 | return OFFLOAD_FAIL(~0); | |||
216 | ||||
217 | TargetMemcpyArgsTy *Args = (TargetMemcpyArgsTy *)Task->shareds; | |||
218 | ||||
219 | if (Args == nullptr) | |||
220 | return OFFLOAD_FAIL(~0); | |||
221 | ||||
222 | // Call blocked version | |||
223 | int Rc = OFFLOAD_SUCCESS(0); | |||
224 | if (Args->IsRectMemcpy) { | |||
225 | Rc = omp_target_memcpy_rect( | |||
226 | Args->Dst, Args->Src, Args->ElementSize, Args->NumDims, Args->Volume, | |||
227 | Args->DstOffsets, Args->SrcOffsets, Args->DstDimensions, | |||
228 | Args->SrcDimensions, Args->DstDevice, Args->SrcDevice); | |||
229 | ||||
230 | DP("omp_target_memcpy_rect returns %d\n", Rc){}; | |||
231 | } else { | |||
232 | Rc = omp_target_memcpy(Args->Dst, Args->Src, Args->Length, Args->DstOffset, | |||
233 | Args->SrcOffset, Args->DstDevice, Args->SrcDevice); | |||
234 | ||||
235 | DP("omp_target_memcpy returns %d\n", Rc){}; | |||
236 | } | |||
237 | ||||
238 | // Release the arguments object | |||
239 | delete Args; | |||
240 | ||||
241 | return Rc; | |||
242 | } | |||
243 | ||||
244 | // Allocate and launch helper task | |||
245 | static int libomp_helper_task_creation(TargetMemcpyArgsTy *Args, | |||
246 | int DepObjCount, | |||
247 | omp_depend_t *DepObjList) { | |||
248 | // Create global thread ID | |||
249 | int Gtid = __kmpc_global_thread_num(nullptr); | |||
250 | int (*Fn)(kmp_int32, kmp_task_t *) = &libomp_target_memcpy_async_helper; | |||
251 | ||||
252 | // Setup the hidden helper flags; | |||
253 | kmp_int32 Flags = 0; | |||
254 | kmp_tasking_flags_t *InputFlags = (kmp_tasking_flags_t *)&Flags; | |||
255 | InputFlags->hidden_helper = 1; | |||
256 | ||||
257 | // Alloc helper task | |||
258 | kmp_task_t *Ptr = __kmpc_omp_target_task_alloc(nullptr, Gtid, Flags, | |||
259 | sizeof(kmp_task_t), 0, Fn, -1); | |||
260 | ||||
261 | if (Ptr == nullptr) { | |||
262 | // Task allocation failed, delete the argument object | |||
263 | delete Args; | |||
264 | ||||
265 | return OFFLOAD_FAIL(~0); | |||
266 | } | |||
267 | ||||
268 | // Setup the arguments passed to helper task | |||
269 | Ptr->shareds = Args; | |||
270 | ||||
271 | // Convert the type of depend objects | |||
272 | llvm::SmallVector<kmp_depend_info_t> DepObjs; | |||
273 | for (int i = 0; i < DepObjCount; i++) { | |||
274 | omp_depend_t DepObj = DepObjList[i]; | |||
275 | DepObjs.push_back(*((kmp_depend_info_t *)DepObj)); | |||
276 | } | |||
277 | ||||
278 | // Launch the helper task | |||
279 | int Rc = __kmpc_omp_task_with_deps(nullptr, Gtid, Ptr, DepObjCount, | |||
280 | DepObjs.data(), 0, nullptr); | |||
281 | ||||
282 | return Rc; | |||
283 | } | |||
284 | ||||
285 | EXTERNextern "C" int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length, | |||
286 | size_t DstOffset, size_t SrcOffset, | |||
287 | int DstDevice, int SrcDevice, | |||
288 | int DepObjCount, omp_depend_t *DepObjList) { | |||
289 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
290 | DP("Call to omp_target_memcpy_async, dst device %d, src device %d, "{} | |||
291 | "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "{} | |||
292 | "src offset %zu, length %zu\n",{} | |||
293 | DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,{} | |||
294 | Length){}; | |||
295 | ||||
296 | // Check the source and dest address | |||
297 | if (Dst == nullptr || Src == nullptr) | |||
298 | return OFFLOAD_FAIL(~0); | |||
299 | ||||
300 | // Create task object | |||
301 | TargetMemcpyArgsTy *Args = new TargetMemcpyArgsTy( | |||
302 | Dst, Src, Length, DstOffset, SrcOffset, DstDevice, SrcDevice); | |||
303 | ||||
304 | // Create and launch helper task | |||
305 | int Rc = libomp_helper_task_creation(Args, DepObjCount, DepObjList); | |||
306 | ||||
307 | DP("omp_target_memcpy_async returns %d\n", Rc){}; | |||
308 | return Rc; | |||
309 | } | |||
310 | ||||
311 | EXTERNextern "C" int | |||
312 | omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize, | |||
313 | int NumDims, const size_t *Volume, | |||
314 | const size_t *DstOffsets, const size_t *SrcOffsets, | |||
315 | const size_t *DstDimensions, const size_t *SrcDimensions, | |||
316 | int DstDevice, int SrcDevice) { | |||
317 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
318 | DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "{} | |||
319 | "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "{} | |||
320 | "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "{} | |||
321 | "volume " DPxMOD ", element size %zu, num_dims %d\n",{} | |||
322 | DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),{} | |||
323 | DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),{} | |||
324 | DPxPTR(Volume), ElementSize, NumDims){}; | |||
325 | ||||
326 | if (!(Dst || Src)) { | |||
327 | DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",{} | |||
328 | INT_MAX){}; | |||
329 | return INT_MAX2147483647; | |||
330 | } | |||
331 | ||||
332 | if (!Dst
| |||
333 | !DstOffsets || !SrcOffsets || !DstDimensions || !SrcDimensions) { | |||
334 | REPORT("Call to omp_target_memcpy_rect with invalid arguments\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Call to omp_target_memcpy_rect with invalid arguments\n"); } while (0);; | |||
335 | return OFFLOAD_FAIL(~0); | |||
336 | } | |||
337 | ||||
338 | int Rc; | |||
339 | if (NumDims == 1) { | |||
340 | Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0], | |||
341 | ElementSize * DstOffsets[0], | |||
342 | ElementSize * SrcOffsets[0], DstDevice, SrcDevice); | |||
343 | } else { | |||
344 | size_t DstSliceSize = ElementSize; | |||
345 | size_t SrcSliceSize = ElementSize; | |||
346 | for (int I = 1; I
| |||
347 | DstSliceSize *= DstDimensions[I]; | |||
348 | SrcSliceSize *= SrcDimensions[I]; | |||
349 | } | |||
350 | ||||
351 | size_t DstOff = DstOffsets[0] * DstSliceSize; | |||
352 | size_t SrcOff = SrcOffsets[0] * SrcSliceSize; | |||
353 | for (size_t I = 0; I < Volume[0]; ++I) { | |||
354 | Rc = omp_target_memcpy_rect( | |||
355 | (char *)Dst + DstOff + DstSliceSize * I, | |||
356 | (char *)const_cast<void *>(Src) + SrcOff + SrcSliceSize * I, | |||
357 | ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1, | |||
358 | DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice); | |||
359 | ||||
360 | if (Rc) { | |||
361 | DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n"){}; | |||
362 | return Rc; | |||
363 | } | |||
364 | } | |||
365 | } | |||
366 | ||||
367 | DP("omp_target_memcpy_rect returns %d\n", Rc){}; | |||
368 | return Rc; | |||
| ||||
369 | } | |||
370 | ||||
371 | EXTERNextern "C" int omp_target_memcpy_rect_async( | |||
372 | void *Dst, const void *Src, size_t ElementSize, int NumDims, | |||
373 | const size_t *Volume, const size_t *DstOffsets, const size_t *SrcOffsets, | |||
374 | const size_t *DstDimensions, const size_t *SrcDimensions, int DstDevice, | |||
375 | int SrcDevice, int DepObjCount, omp_depend_t *DepObjList) { | |||
376 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
377 | DP("Call to omp_target_memcpy_rect_async, dst device %d, src device %d, "{} | |||
378 | "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "{} | |||
379 | "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "{} | |||
380 | "volume " DPxMOD ", element size %zu, num_dims %d\n",{} | |||
381 | DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),{} | |||
382 | DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),{} | |||
383 | DPxPTR(Volume), ElementSize, NumDims){}; | |||
384 | ||||
385 | // Need to check this first to not return OFFLOAD_FAIL instead | |||
386 | if (!Dst && !Src) { | |||
387 | DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",{} | |||
388 | INT_MAX){}; | |||
389 | return INT_MAX2147483647; | |||
390 | } | |||
391 | ||||
392 | // Check the source and dest address | |||
393 | if (Dst == nullptr || Src == nullptr) | |||
394 | return OFFLOAD_FAIL(~0); | |||
395 | ||||
396 | // Create task object | |||
397 | TargetMemcpyArgsTy *Args = new TargetMemcpyArgsTy( | |||
398 | Dst, Src, ElementSize, NumDims, Volume, DstOffsets, SrcOffsets, | |||
399 | DstDimensions, SrcDimensions, DstDevice, SrcDevice); | |||
400 | ||||
401 | // Create and launch helper task | |||
402 | int Rc = libomp_helper_task_creation(Args, DepObjCount, DepObjList); | |||
403 | ||||
404 | DP("omp_target_memcpy_rect_async returns %d\n", Rc){}; | |||
405 | return Rc; | |||
406 | } | |||
407 | ||||
408 | EXTERNextern "C" int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr, | |||
409 | size_t Size, size_t DeviceOffset, | |||
410 | int DeviceNum) { | |||
411 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
412 | DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "{} | |||
413 | "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",{} | |||
414 | DPxPTR(HostPtr), DPxPTR(DevicePtr), Size, DeviceOffset, DeviceNum){}; | |||
415 | ||||
416 | if (!HostPtr || !DevicePtr || Size <= 0) { | |||
417 | REPORT("Call to omp_target_associate_ptr with invalid arguments\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Call to omp_target_associate_ptr with invalid arguments\n" ); } while (0);; | |||
418 | return OFFLOAD_FAIL(~0); | |||
419 | } | |||
420 | ||||
421 | if (DeviceNum == omp_get_initial_device()) { | |||
422 | REPORT("omp_target_associate_ptr: no association possible on the host\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_associate_ptr: no association possible on the host\n" ); } while (0);; | |||
423 | return OFFLOAD_FAIL(~0); | |||
424 | } | |||
425 | ||||
426 | if (!deviceIsReady(DeviceNum)) { | |||
427 | REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_associate_ptr returns OFFLOAD_FAIL\n"); } while (0);; | |||
428 | return OFFLOAD_FAIL(~0); | |||
429 | } | |||
430 | ||||
431 | DeviceTy &Device = *PM->Devices[DeviceNum]; | |||
432 | void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset); | |||
433 | int Rc = Device.associatePtr(const_cast<void *>(HostPtr), | |||
434 | const_cast<void *>(DeviceAddr), Size); | |||
435 | DP("omp_target_associate_ptr returns %d\n", Rc){}; | |||
436 | return Rc; | |||
437 | } | |||
438 | ||||
439 | EXTERNextern "C" int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) { | |||
440 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
441 | DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "{} | |||
442 | "device_num %d\n",{} | |||
443 | DPxPTR(HostPtr), DeviceNum){}; | |||
444 | ||||
445 | if (!HostPtr) { | |||
446 | REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Call to omp_target_associate_ptr with invalid host_ptr\n") ; } while (0);; | |||
447 | return OFFLOAD_FAIL(~0); | |||
448 | } | |||
449 | ||||
450 | if (DeviceNum == omp_get_initial_device()) { | |||
451 | REPORT(do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_disassociate_ptr: no association possible on the host\n" ); } while (0); | |||
452 | "omp_target_disassociate_ptr: no association possible on the host\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_disassociate_ptr: no association possible on the host\n" ); } while (0);; | |||
453 | return OFFLOAD_FAIL(~0); | |||
454 | } | |||
455 | ||||
456 | if (!deviceIsReady(DeviceNum)) { | |||
457 | REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "omp_target_disassociate_ptr returns OFFLOAD_FAIL\n"); } while (0);; | |||
458 | return OFFLOAD_FAIL(~0); | |||
459 | } | |||
460 | ||||
461 | DeviceTy &Device = *PM->Devices[DeviceNum]; | |||
462 | int Rc = Device.disassociatePtr(const_cast<void *>(HostPtr)); | |||
463 | DP("omp_target_disassociate_ptr returns %d\n", Rc){}; | |||
464 | return Rc; | |||
465 | } | |||
466 | ||||
467 | EXTERNextern "C" void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) { | |||
468 | TIMESCOPE()llvm::TimeTraceScope TimeScope(__FUNCTION__); | |||
469 | DP("Call to omp_get_mapped_ptr with ptr " DPxMOD ", device_num %d.\n",{} | |||
470 | DPxPTR(Ptr), DeviceNum){}; | |||
471 | ||||
472 | if (!Ptr) { | |||
473 | REPORT("Call to omp_get_mapped_ptr with nullptr.\n")do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Call to omp_get_mapped_ptr with nullptr.\n"); } while (0);; | |||
474 | return nullptr; | |||
475 | } | |||
476 | ||||
477 | if (DeviceNum == omp_get_initial_device()) { | |||
478 | REPORT("Device %d is initial device, returning Ptr " DPxMOD ".\n",do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Device %d is initial device, returning Ptr " "0x%0*" "l" "x" ".\n", DeviceNum, ((int)(2 * sizeof(uintptr_t))), ((uintptr_t )(Ptr))); } while (0); | |||
479 | DeviceNum, DPxPTR(Ptr))do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Device %d is initial device, returning Ptr " "0x%0*" "l" "x" ".\n", DeviceNum, ((int)(2 * sizeof(uintptr_t))), ((uintptr_t )(Ptr))); } while (0);; | |||
480 | return const_cast<void *>(Ptr); | |||
481 | } | |||
482 | ||||
483 | int DevicesSize = omp_get_initial_device(); | |||
484 | { | |||
485 | std::lock_guard<std::mutex> LG(PM->RTLsMtx); | |||
486 | DevicesSize = PM->Devices.size(); | |||
487 | } | |||
488 | if (DevicesSize <= DeviceNum) { | |||
489 | DP("DeviceNum %d is invalid, returning nullptr.\n", DeviceNum){}; | |||
490 | return nullptr; | |||
491 | } | |||
492 | ||||
493 | if (!deviceIsReady(DeviceNum)) { | |||
494 | REPORT("Device %d is not ready, returning nullptr.\n", DeviceNum)do { fprintf(stderr, "Libomptarget" " error: "); fprintf(stderr , "Device %d is not ready, returning nullptr.\n", DeviceNum); } while (0);; | |||
495 | return nullptr; | |||
496 | } | |||
497 | ||||
498 | auto &Device = *PM->Devices[DeviceNum]; | |||
499 | TargetPointerResultTy TPR = Device.getTgtPtrBegin(const_cast<void *>(Ptr), 1, | |||
500 | /*UpdateRefCount=*/false, | |||
501 | /*UseHoldRefCount=*/false); | |||
502 | if (!TPR.isPresent()) { | |||
503 | DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n",{} | |||
504 | DPxPTR(Ptr), DeviceNum){}; | |||
505 | return nullptr; | |||
506 | } | |||
507 | ||||
508 | DP("omp_get_mapped_ptr returns " DPxMOD ".\n", DPxPTR(TPR.TargetPointer)){}; | |||
509 | ||||
510 | return TPR.TargetPointer; | |||
511 | } |