File: | llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp |
Warning: | line 1140, column 32 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | // | ||||||
9 | /// \file | ||||||
10 | /// This file is a part of MemorySanitizer, a detector of uninitialized | ||||||
11 | /// reads. | ||||||
12 | /// | ||||||
13 | /// The algorithm of the tool is similar to Memcheck | ||||||
14 | /// (http://goo.gl/QKbem). We associate a few shadow bits with every | ||||||
15 | /// byte of the application memory, poison the shadow of the malloc-ed | ||||||
16 | /// or alloca-ed memory, load the shadow bits on every memory read, | ||||||
17 | /// propagate the shadow bits through some of the arithmetic | ||||||
18 | /// instruction (including MOV), store the shadow bits on every memory | ||||||
19 | /// write, report a bug on some other instructions (e.g. JMP) if the | ||||||
20 | /// associated shadow is poisoned. | ||||||
21 | /// | ||||||
22 | /// But there are differences too. The first and the major one: | ||||||
23 | /// compiler instrumentation instead of binary instrumentation. This | ||||||
24 | /// gives us much better register allocation, possible compiler | ||||||
25 | /// optimizations and a fast start-up. But this brings the major issue | ||||||
26 | /// as well: msan needs to see all program events, including system | ||||||
27 | /// calls and reads/writes in system libraries, so we either need to | ||||||
28 | /// compile *everything* with msan or use a binary translation | ||||||
29 | /// component (e.g. DynamoRIO) to instrument pre-built libraries. | ||||||
30 | /// Another difference from Memcheck is that we use 8 shadow bits per | ||||||
31 | /// byte of application memory and use a direct shadow mapping. This | ||||||
32 | /// greatly simplifies the instrumentation code and avoids races on | ||||||
33 | /// shadow updates (Memcheck is single-threaded so races are not a | ||||||
34 | /// concern there. Memcheck uses 2 shadow bits per byte with a slow | ||||||
35 | /// path storage that uses 8 bits per byte). | ||||||
36 | /// | ||||||
37 | /// The default value of shadow is 0, which means "clean" (not poisoned). | ||||||
38 | /// | ||||||
39 | /// Every module initializer should call __msan_init to ensure that the | ||||||
40 | /// shadow memory is ready. On error, __msan_warning is called. Since | ||||||
41 | /// parameters and return values may be passed via registers, we have a | ||||||
42 | /// specialized thread-local shadow for return values | ||||||
43 | /// (__msan_retval_tls) and parameters (__msan_param_tls). | ||||||
44 | /// | ||||||
45 | /// Origin tracking. | ||||||
46 | /// | ||||||
47 | /// MemorySanitizer can track origins (allocation points) of all uninitialized | ||||||
48 | /// values. This behavior is controlled with a flag (msan-track-origins) and is | ||||||
49 | /// disabled by default. | ||||||
50 | /// | ||||||
51 | /// Origins are 4-byte values created and interpreted by the runtime library. | ||||||
52 | /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes | ||||||
53 | /// of application memory. Propagation of origins is basically a bunch of | ||||||
54 | /// "select" instructions that pick the origin of a dirty argument, if an | ||||||
55 | /// instruction has one. | ||||||
56 | /// | ||||||
57 | /// Every 4 aligned, consecutive bytes of application memory have one origin | ||||||
58 | /// value associated with them. If these bytes contain uninitialized data | ||||||
59 | /// coming from 2 different allocations, the last store wins. Because of this, | ||||||
60 | /// MemorySanitizer reports can show unrelated origins, but this is unlikely in | ||||||
61 | /// practice. | ||||||
62 | /// | ||||||
63 | /// Origins are meaningless for fully initialized values, so MemorySanitizer | ||||||
64 | /// avoids storing origin to memory when a fully initialized value is stored. | ||||||
65 | /// This way it avoids needless overwritting origin of the 4-byte region on | ||||||
66 | /// a short (i.e. 1 byte) clean store, and it is also good for performance. | ||||||
67 | /// | ||||||
68 | /// Atomic handling. | ||||||
69 | /// | ||||||
70 | /// Ideally, every atomic store of application value should update the | ||||||
71 | /// corresponding shadow location in an atomic way. Unfortunately, atomic store | ||||||
72 | /// of two disjoint locations can not be done without severe slowdown. | ||||||
73 | /// | ||||||
74 | /// Therefore, we implement an approximation that may err on the safe side. | ||||||
75 | /// In this implementation, every atomically accessed location in the program | ||||||
76 | /// may only change from (partially) uninitialized to fully initialized, but | ||||||
77 | /// not the other way around. We load the shadow _after_ the application load, | ||||||
78 | /// and we store the shadow _before_ the app store. Also, we always store clean | ||||||
79 | /// shadow (if the application store is atomic). This way, if the store-load | ||||||
80 | /// pair constitutes a happens-before arc, shadow store and load are correctly | ||||||
81 | /// ordered such that the load will get either the value that was stored, or | ||||||
82 | /// some later value (which is always clean). | ||||||
83 | /// | ||||||
84 | /// This does not work very well with Compare-And-Swap (CAS) and | ||||||
85 | /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW | ||||||
86 | /// must store the new shadow before the app operation, and load the shadow | ||||||
87 | /// after the app operation. Computers don't work this way. Current | ||||||
88 | /// implementation ignores the load aspect of CAS/RMW, always returning a clean | ||||||
89 | /// value. It implements the store part as a simple atomic store by storing a | ||||||
90 | /// clean shadow. | ||||||
91 | /// | ||||||
92 | /// Instrumenting inline assembly. | ||||||
93 | /// | ||||||
94 | /// For inline assembly code LLVM has little idea about which memory locations | ||||||
95 | /// become initialized depending on the arguments. It can be possible to figure | ||||||
96 | /// out which arguments are meant to point to inputs and outputs, but the | ||||||
97 | /// actual semantics can be only visible at runtime. In the Linux kernel it's | ||||||
98 | /// also possible that the arguments only indicate the offset for a base taken | ||||||
99 | /// from a segment register, so it's dangerous to treat any asm() arguments as | ||||||
100 | /// pointers. We take a conservative approach generating calls to | ||||||
101 | /// __msan_instrument_asm_store(ptr, size) | ||||||
102 | /// , which defer the memory unpoisoning to the runtime library. | ||||||
103 | /// The latter can perform more complex address checks to figure out whether | ||||||
104 | /// it's safe to touch the shadow memory. | ||||||
105 | /// Like with atomic operations, we call __msan_instrument_asm_store() before | ||||||
106 | /// the assembly call, so that changes to the shadow memory will be seen by | ||||||
107 | /// other threads together with main memory initialization. | ||||||
108 | /// | ||||||
109 | /// KernelMemorySanitizer (KMSAN) implementation. | ||||||
110 | /// | ||||||
111 | /// The major differences between KMSAN and MSan instrumentation are: | ||||||
112 | /// - KMSAN always tracks the origins and implies msan-keep-going=true; | ||||||
113 | /// - KMSAN allocates shadow and origin memory for each page separately, so | ||||||
114 | /// there are no explicit accesses to shadow and origin in the | ||||||
115 | /// instrumentation. | ||||||
116 | /// Shadow and origin values for a particular X-byte memory location | ||||||
117 | /// (X=1,2,4,8) are accessed through pointers obtained via the | ||||||
118 | /// __msan_metadata_ptr_for_load_X(ptr) | ||||||
119 | /// __msan_metadata_ptr_for_store_X(ptr) | ||||||
120 | /// functions. The corresponding functions check that the X-byte accesses | ||||||
121 | /// are possible and returns the pointers to shadow and origin memory. | ||||||
122 | /// Arbitrary sized accesses are handled with: | ||||||
123 | /// __msan_metadata_ptr_for_load_n(ptr, size) | ||||||
124 | /// __msan_metadata_ptr_for_store_n(ptr, size); | ||||||
125 | /// - TLS variables are stored in a single per-task struct. A call to a | ||||||
126 | /// function __msan_get_context_state() returning a pointer to that struct | ||||||
127 | /// is inserted into every instrumented function before the entry block; | ||||||
128 | /// - __msan_warning() takes a 32-bit origin parameter; | ||||||
129 | /// - local variables are poisoned with __msan_poison_alloca() upon function | ||||||
130 | /// entry and unpoisoned with __msan_unpoison_alloca() before leaving the | ||||||
131 | /// function; | ||||||
132 | /// - the pass doesn't declare any global variables or add global constructors | ||||||
133 | /// to the translation unit. | ||||||
134 | /// | ||||||
135 | /// Also, KMSAN currently ignores uninitialized memory passed into inline asm | ||||||
136 | /// calls, making sure we're on the safe side wrt. possible false positives. | ||||||
137 | /// | ||||||
138 | /// KernelMemorySanitizer only supports X86_64 at the moment. | ||||||
139 | /// | ||||||
140 | //===----------------------------------------------------------------------===// | ||||||
141 | |||||||
142 | #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" | ||||||
143 | #include "llvm/ADT/APInt.h" | ||||||
144 | #include "llvm/ADT/ArrayRef.h" | ||||||
145 | #include "llvm/ADT/DepthFirstIterator.h" | ||||||
146 | #include "llvm/ADT/SmallSet.h" | ||||||
147 | #include "llvm/ADT/SmallString.h" | ||||||
148 | #include "llvm/ADT/SmallVector.h" | ||||||
149 | #include "llvm/ADT/StringExtras.h" | ||||||
150 | #include "llvm/ADT/StringRef.h" | ||||||
151 | #include "llvm/ADT/Triple.h" | ||||||
152 | #include "llvm/Analysis/TargetLibraryInfo.h" | ||||||
153 | #include "llvm/IR/Argument.h" | ||||||
154 | #include "llvm/IR/Attributes.h" | ||||||
155 | #include "llvm/IR/BasicBlock.h" | ||||||
156 | #include "llvm/IR/CallSite.h" | ||||||
157 | #include "llvm/IR/CallingConv.h" | ||||||
158 | #include "llvm/IR/Constant.h" | ||||||
159 | #include "llvm/IR/Constants.h" | ||||||
160 | #include "llvm/IR/DataLayout.h" | ||||||
161 | #include "llvm/IR/DerivedTypes.h" | ||||||
162 | #include "llvm/IR/Function.h" | ||||||
163 | #include "llvm/IR/GlobalValue.h" | ||||||
164 | #include "llvm/IR/GlobalVariable.h" | ||||||
165 | #include "llvm/IR/IRBuilder.h" | ||||||
166 | #include "llvm/IR/InlineAsm.h" | ||||||
167 | #include "llvm/IR/InstVisitor.h" | ||||||
168 | #include "llvm/IR/InstrTypes.h" | ||||||
169 | #include "llvm/IR/Instruction.h" | ||||||
170 | #include "llvm/IR/Instructions.h" | ||||||
171 | #include "llvm/IR/IntrinsicInst.h" | ||||||
172 | #include "llvm/IR/Intrinsics.h" | ||||||
173 | #include "llvm/IR/IntrinsicsX86.h" | ||||||
174 | #include "llvm/IR/LLVMContext.h" | ||||||
175 | #include "llvm/IR/MDBuilder.h" | ||||||
176 | #include "llvm/IR/Module.h" | ||||||
177 | #include "llvm/IR/Type.h" | ||||||
178 | #include "llvm/IR/Value.h" | ||||||
179 | #include "llvm/IR/ValueMap.h" | ||||||
180 | #include "llvm/InitializePasses.h" | ||||||
181 | #include "llvm/Pass.h" | ||||||
182 | #include "llvm/Support/AtomicOrdering.h" | ||||||
183 | #include "llvm/Support/Casting.h" | ||||||
184 | #include "llvm/Support/CommandLine.h" | ||||||
185 | #include "llvm/Support/Compiler.h" | ||||||
186 | #include "llvm/Support/Debug.h" | ||||||
187 | #include "llvm/Support/ErrorHandling.h" | ||||||
188 | #include "llvm/Support/MathExtras.h" | ||||||
189 | #include "llvm/Support/raw_ostream.h" | ||||||
190 | #include "llvm/Transforms/Instrumentation.h" | ||||||
191 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | ||||||
192 | #include "llvm/Transforms/Utils/Local.h" | ||||||
193 | #include "llvm/Transforms/Utils/ModuleUtils.h" | ||||||
194 | #include <algorithm> | ||||||
195 | #include <cassert> | ||||||
196 | #include <cstddef> | ||||||
197 | #include <cstdint> | ||||||
198 | #include <memory> | ||||||
199 | #include <string> | ||||||
200 | #include <tuple> | ||||||
201 | |||||||
202 | using namespace llvm; | ||||||
203 | |||||||
204 | #define DEBUG_TYPE"msan" "msan" | ||||||
205 | |||||||
206 | static const unsigned kOriginSize = 4; | ||||||
207 | static const Align kMinOriginAlignment = Align(4); | ||||||
208 | static const Align kShadowTLSAlignment = Align(8); | ||||||
209 | |||||||
210 | // These constants must be kept in sync with the ones in msan.h. | ||||||
211 | static const unsigned kParamTLSSize = 800; | ||||||
212 | static const unsigned kRetvalTLSSize = 800; | ||||||
213 | |||||||
214 | // Accesses sizes are powers of two: 1, 2, 4, 8. | ||||||
215 | static const size_t kNumberOfAccessSizes = 4; | ||||||
216 | |||||||
217 | /// Track origins of uninitialized values. | ||||||
218 | /// | ||||||
219 | /// Adds a section to MemorySanitizer report that points to the allocation | ||||||
220 | /// (stack or heap) the uninitialized bits came from originally. | ||||||
221 | static cl::opt<int> ClTrackOrigins("msan-track-origins", | ||||||
222 | cl::desc("Track origins (allocation sites) of poisoned memory"), | ||||||
223 | cl::Hidden, cl::init(0)); | ||||||
224 | |||||||
225 | static cl::opt<bool> ClKeepGoing("msan-keep-going", | ||||||
226 | cl::desc("keep going after reporting a UMR"), | ||||||
227 | cl::Hidden, cl::init(false)); | ||||||
228 | |||||||
229 | static cl::opt<bool> ClPoisonStack("msan-poison-stack", | ||||||
230 | cl::desc("poison uninitialized stack variables"), | ||||||
231 | cl::Hidden, cl::init(true)); | ||||||
232 | |||||||
233 | static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call", | ||||||
234 | cl::desc("poison uninitialized stack variables with a call"), | ||||||
235 | cl::Hidden, cl::init(false)); | ||||||
236 | |||||||
237 | static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern", | ||||||
238 | cl::desc("poison uninitialized stack variables with the given pattern"), | ||||||
239 | cl::Hidden, cl::init(0xff)); | ||||||
240 | |||||||
241 | static cl::opt<bool> ClPoisonUndef("msan-poison-undef", | ||||||
242 | cl::desc("poison undef temps"), | ||||||
243 | cl::Hidden, cl::init(true)); | ||||||
244 | |||||||
245 | static cl::opt<bool> ClHandleICmp("msan-handle-icmp", | ||||||
246 | cl::desc("propagate shadow through ICmpEQ and ICmpNE"), | ||||||
247 | cl::Hidden, cl::init(true)); | ||||||
248 | |||||||
249 | static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact", | ||||||
250 | cl::desc("exact handling of relational integer ICmp"), | ||||||
251 | cl::Hidden, cl::init(false)); | ||||||
252 | |||||||
253 | static cl::opt<bool> ClHandleLifetimeIntrinsics( | ||||||
254 | "msan-handle-lifetime-intrinsics", | ||||||
255 | cl::desc( | ||||||
256 | "when possible, poison scoped variables at the beginning of the scope " | ||||||
257 | "(slower, but more precise)"), | ||||||
258 | cl::Hidden, cl::init(true)); | ||||||
259 | |||||||
260 | // When compiling the Linux kernel, we sometimes see false positives related to | ||||||
261 | // MSan being unable to understand that inline assembly calls may initialize | ||||||
262 | // local variables. | ||||||
263 | // This flag makes the compiler conservatively unpoison every memory location | ||||||
264 | // passed into an assembly call. Note that this may cause false positives. | ||||||
265 | // Because it's impossible to figure out the array sizes, we can only unpoison | ||||||
266 | // the first sizeof(type) bytes for each type* pointer. | ||||||
267 | // The instrumentation is only enabled in KMSAN builds, and only if | ||||||
268 | // -msan-handle-asm-conservative is on. This is done because we may want to | ||||||
269 | // quickly disable assembly instrumentation when it breaks. | ||||||
270 | static cl::opt<bool> ClHandleAsmConservative( | ||||||
271 | "msan-handle-asm-conservative", | ||||||
272 | cl::desc("conservative handling of inline assembly"), cl::Hidden, | ||||||
273 | cl::init(true)); | ||||||
274 | |||||||
275 | // This flag controls whether we check the shadow of the address | ||||||
276 | // operand of load or store. Such bugs are very rare, since load from | ||||||
277 | // a garbage address typically results in SEGV, but still happen | ||||||
278 | // (e.g. only lower bits of address are garbage, or the access happens | ||||||
279 | // early at program startup where malloc-ed memory is more likely to | ||||||
280 | // be zeroed. As of 2012-08-28 this flag adds 20% slowdown. | ||||||
281 | static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address", | ||||||
282 | cl::desc("report accesses through a pointer which has poisoned shadow"), | ||||||
283 | cl::Hidden, cl::init(true)); | ||||||
284 | |||||||
285 | static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions", | ||||||
286 | cl::desc("print out instructions with default strict semantics"), | ||||||
287 | cl::Hidden, cl::init(false)); | ||||||
288 | |||||||
289 | static cl::opt<int> ClInstrumentationWithCallThreshold( | ||||||
290 | "msan-instrumentation-with-call-threshold", | ||||||
291 | cl::desc( | ||||||
292 | "If the function being instrumented requires more than " | ||||||
293 | "this number of checks and origin stores, use callbacks instead of " | ||||||
294 | "inline checks (-1 means never use callbacks)."), | ||||||
295 | cl::Hidden, cl::init(3500)); | ||||||
296 | |||||||
297 | static cl::opt<bool> | ||||||
298 | ClEnableKmsan("msan-kernel", | ||||||
299 | cl::desc("Enable KernelMemorySanitizer instrumentation"), | ||||||
300 | cl::Hidden, cl::init(false)); | ||||||
301 | |||||||
302 | // This is an experiment to enable handling of cases where shadow is a non-zero | ||||||
303 | // compile-time constant. For some unexplainable reason they were silently | ||||||
304 | // ignored in the instrumentation. | ||||||
305 | static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow", | ||||||
306 | cl::desc("Insert checks for constant shadow values"), | ||||||
307 | cl::Hidden, cl::init(false)); | ||||||
308 | |||||||
309 | // This is off by default because of a bug in gold: | ||||||
310 | // https://sourceware.org/bugzilla/show_bug.cgi?id=19002 | ||||||
311 | static cl::opt<bool> ClWithComdat("msan-with-comdat", | ||||||
312 | cl::desc("Place MSan constructors in comdat sections"), | ||||||
313 | cl::Hidden, cl::init(false)); | ||||||
314 | |||||||
315 | // These options allow to specify custom memory map parameters | ||||||
316 | // See MemoryMapParams for details. | ||||||
317 | static cl::opt<uint64_t> ClAndMask("msan-and-mask", | ||||||
318 | cl::desc("Define custom MSan AndMask"), | ||||||
319 | cl::Hidden, cl::init(0)); | ||||||
320 | |||||||
321 | static cl::opt<uint64_t> ClXorMask("msan-xor-mask", | ||||||
322 | cl::desc("Define custom MSan XorMask"), | ||||||
323 | cl::Hidden, cl::init(0)); | ||||||
324 | |||||||
325 | static cl::opt<uint64_t> ClShadowBase("msan-shadow-base", | ||||||
326 | cl::desc("Define custom MSan ShadowBase"), | ||||||
327 | cl::Hidden, cl::init(0)); | ||||||
328 | |||||||
329 | static cl::opt<uint64_t> ClOriginBase("msan-origin-base", | ||||||
330 | cl::desc("Define custom MSan OriginBase"), | ||||||
331 | cl::Hidden, cl::init(0)); | ||||||
332 | |||||||
333 | static const char *const kMsanModuleCtorName = "msan.module_ctor"; | ||||||
334 | static const char *const kMsanInitName = "__msan_init"; | ||||||
335 | |||||||
336 | namespace { | ||||||
337 | |||||||
338 | // Memory map parameters used in application-to-shadow address calculation. | ||||||
339 | // Offset = (Addr & ~AndMask) ^ XorMask | ||||||
340 | // Shadow = ShadowBase + Offset | ||||||
341 | // Origin = OriginBase + Offset | ||||||
342 | struct MemoryMapParams { | ||||||
343 | uint64_t AndMask; | ||||||
344 | uint64_t XorMask; | ||||||
345 | uint64_t ShadowBase; | ||||||
346 | uint64_t OriginBase; | ||||||
347 | }; | ||||||
348 | |||||||
349 | struct PlatformMemoryMapParams { | ||||||
350 | const MemoryMapParams *bits32; | ||||||
351 | const MemoryMapParams *bits64; | ||||||
352 | }; | ||||||
353 | |||||||
354 | } // end anonymous namespace | ||||||
355 | |||||||
356 | // i386 Linux | ||||||
357 | static const MemoryMapParams Linux_I386_MemoryMapParams = { | ||||||
358 | 0x000080000000, // AndMask | ||||||
359 | 0, // XorMask (not used) | ||||||
360 | 0, // ShadowBase (not used) | ||||||
361 | 0x000040000000, // OriginBase | ||||||
362 | }; | ||||||
363 | |||||||
364 | // x86_64 Linux | ||||||
365 | static const MemoryMapParams Linux_X86_64_MemoryMapParams = { | ||||||
366 | #ifdef MSAN_LINUX_X86_64_OLD_MAPPING | ||||||
367 | 0x400000000000, // AndMask | ||||||
368 | 0, // XorMask (not used) | ||||||
369 | 0, // ShadowBase (not used) | ||||||
370 | 0x200000000000, // OriginBase | ||||||
371 | #else | ||||||
372 | 0, // AndMask (not used) | ||||||
373 | 0x500000000000, // XorMask | ||||||
374 | 0, // ShadowBase (not used) | ||||||
375 | 0x100000000000, // OriginBase | ||||||
376 | #endif | ||||||
377 | }; | ||||||
378 | |||||||
379 | // mips64 Linux | ||||||
380 | static const MemoryMapParams Linux_MIPS64_MemoryMapParams = { | ||||||
381 | 0, // AndMask (not used) | ||||||
382 | 0x008000000000, // XorMask | ||||||
383 | 0, // ShadowBase (not used) | ||||||
384 | 0x002000000000, // OriginBase | ||||||
385 | }; | ||||||
386 | |||||||
387 | // ppc64 Linux | ||||||
388 | static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = { | ||||||
389 | 0xE00000000000, // AndMask | ||||||
390 | 0x100000000000, // XorMask | ||||||
391 | 0x080000000000, // ShadowBase | ||||||
392 | 0x1C0000000000, // OriginBase | ||||||
393 | }; | ||||||
394 | |||||||
395 | // aarch64 Linux | ||||||
396 | static const MemoryMapParams Linux_AArch64_MemoryMapParams = { | ||||||
397 | 0, // AndMask (not used) | ||||||
398 | 0x06000000000, // XorMask | ||||||
399 | 0, // ShadowBase (not used) | ||||||
400 | 0x01000000000, // OriginBase | ||||||
401 | }; | ||||||
402 | |||||||
403 | // i386 FreeBSD | ||||||
404 | static const MemoryMapParams FreeBSD_I386_MemoryMapParams = { | ||||||
405 | 0x000180000000, // AndMask | ||||||
406 | 0x000040000000, // XorMask | ||||||
407 | 0x000020000000, // ShadowBase | ||||||
408 | 0x000700000000, // OriginBase | ||||||
409 | }; | ||||||
410 | |||||||
411 | // x86_64 FreeBSD | ||||||
412 | static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = { | ||||||
413 | 0xc00000000000, // AndMask | ||||||
414 | 0x200000000000, // XorMask | ||||||
415 | 0x100000000000, // ShadowBase | ||||||
416 | 0x380000000000, // OriginBase | ||||||
417 | }; | ||||||
418 | |||||||
419 | // x86_64 NetBSD | ||||||
420 | static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = { | ||||||
421 | 0, // AndMask | ||||||
422 | 0x500000000000, // XorMask | ||||||
423 | 0, // ShadowBase | ||||||
424 | 0x100000000000, // OriginBase | ||||||
425 | }; | ||||||
426 | |||||||
427 | static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = { | ||||||
428 | &Linux_I386_MemoryMapParams, | ||||||
429 | &Linux_X86_64_MemoryMapParams, | ||||||
430 | }; | ||||||
431 | |||||||
432 | static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = { | ||||||
433 | nullptr, | ||||||
434 | &Linux_MIPS64_MemoryMapParams, | ||||||
435 | }; | ||||||
436 | |||||||
437 | static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = { | ||||||
438 | nullptr, | ||||||
439 | &Linux_PowerPC64_MemoryMapParams, | ||||||
440 | }; | ||||||
441 | |||||||
442 | static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = { | ||||||
443 | nullptr, | ||||||
444 | &Linux_AArch64_MemoryMapParams, | ||||||
445 | }; | ||||||
446 | |||||||
447 | static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = { | ||||||
448 | &FreeBSD_I386_MemoryMapParams, | ||||||
449 | &FreeBSD_X86_64_MemoryMapParams, | ||||||
450 | }; | ||||||
451 | |||||||
452 | static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = { | ||||||
453 | nullptr, | ||||||
454 | &NetBSD_X86_64_MemoryMapParams, | ||||||
455 | }; | ||||||
456 | |||||||
457 | namespace { | ||||||
458 | |||||||
459 | /// Instrument functions of a module to detect uninitialized reads. | ||||||
460 | /// | ||||||
461 | /// Instantiating MemorySanitizer inserts the msan runtime library API function | ||||||
462 | /// declarations into the module if they don't exist already. Instantiating | ||||||
463 | /// ensures the __msan_init function is in the list of global constructors for | ||||||
464 | /// the module. | ||||||
465 | class MemorySanitizer { | ||||||
466 | public: | ||||||
467 | MemorySanitizer(Module &M, MemorySanitizerOptions Options) | ||||||
468 | : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins), | ||||||
469 | Recover(Options.Recover) { | ||||||
470 | initializeModule(M); | ||||||
471 | } | ||||||
472 | |||||||
473 | // MSan cannot be moved or copied because of MapParams. | ||||||
474 | MemorySanitizer(MemorySanitizer &&) = delete; | ||||||
475 | MemorySanitizer &operator=(MemorySanitizer &&) = delete; | ||||||
476 | MemorySanitizer(const MemorySanitizer &) = delete; | ||||||
477 | MemorySanitizer &operator=(const MemorySanitizer &) = delete; | ||||||
478 | |||||||
479 | bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI); | ||||||
480 | |||||||
481 | private: | ||||||
482 | friend struct MemorySanitizerVisitor; | ||||||
483 | friend struct VarArgAMD64Helper; | ||||||
484 | friend struct VarArgMIPS64Helper; | ||||||
485 | friend struct VarArgAArch64Helper; | ||||||
486 | friend struct VarArgPowerPC64Helper; | ||||||
487 | |||||||
488 | void initializeModule(Module &M); | ||||||
489 | void initializeCallbacks(Module &M); | ||||||
490 | void createKernelApi(Module &M); | ||||||
491 | void createUserspaceApi(Module &M); | ||||||
492 | |||||||
493 | /// True if we're compiling the Linux kernel. | ||||||
494 | bool CompileKernel; | ||||||
495 | /// Track origins (allocation points) of uninitialized values. | ||||||
496 | int TrackOrigins; | ||||||
497 | bool Recover; | ||||||
498 | |||||||
499 | LLVMContext *C; | ||||||
500 | Type *IntptrTy; | ||||||
501 | Type *OriginTy; | ||||||
502 | |||||||
503 | // XxxTLS variables represent the per-thread state in MSan and per-task state | ||||||
504 | // in KMSAN. | ||||||
505 | // For the userspace these point to thread-local globals. In the kernel land | ||||||
506 | // they point to the members of a per-task struct obtained via a call to | ||||||
507 | // __msan_get_context_state(). | ||||||
508 | |||||||
509 | /// Thread-local shadow storage for function parameters. | ||||||
510 | Value *ParamTLS; | ||||||
511 | |||||||
512 | /// Thread-local origin storage for function parameters. | ||||||
513 | Value *ParamOriginTLS; | ||||||
514 | |||||||
515 | /// Thread-local shadow storage for function return value. | ||||||
516 | Value *RetvalTLS; | ||||||
517 | |||||||
518 | /// Thread-local origin storage for function return value. | ||||||
519 | Value *RetvalOriginTLS; | ||||||
520 | |||||||
521 | /// Thread-local shadow storage for in-register va_arg function | ||||||
522 | /// parameters (x86_64-specific). | ||||||
523 | Value *VAArgTLS; | ||||||
524 | |||||||
525 | /// Thread-local shadow storage for in-register va_arg function | ||||||
526 | /// parameters (x86_64-specific). | ||||||
527 | Value *VAArgOriginTLS; | ||||||
528 | |||||||
529 | /// Thread-local shadow storage for va_arg overflow area | ||||||
530 | /// (x86_64-specific). | ||||||
531 | Value *VAArgOverflowSizeTLS; | ||||||
532 | |||||||
533 | /// Thread-local space used to pass origin value to the UMR reporting | ||||||
534 | /// function. | ||||||
535 | Value *OriginTLS; | ||||||
536 | |||||||
537 | /// Are the instrumentation callbacks set up? | ||||||
538 | bool CallbacksInitialized = false; | ||||||
539 | |||||||
540 | /// The run-time callback to print a warning. | ||||||
541 | FunctionCallee WarningFn; | ||||||
542 | |||||||
543 | // These arrays are indexed by log2(AccessSize). | ||||||
544 | FunctionCallee MaybeWarningFn[kNumberOfAccessSizes]; | ||||||
545 | FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes]; | ||||||
546 | |||||||
547 | /// Run-time helper that generates a new origin value for a stack | ||||||
548 | /// allocation. | ||||||
549 | FunctionCallee MsanSetAllocaOrigin4Fn; | ||||||
550 | |||||||
551 | /// Run-time helper that poisons stack on function entry. | ||||||
552 | FunctionCallee MsanPoisonStackFn; | ||||||
553 | |||||||
554 | /// Run-time helper that records a store (or any event) of an | ||||||
555 | /// uninitialized value and returns an updated origin id encoding this info. | ||||||
556 | FunctionCallee MsanChainOriginFn; | ||||||
557 | |||||||
558 | /// MSan runtime replacements for memmove, memcpy and memset. | ||||||
559 | FunctionCallee MemmoveFn, MemcpyFn, MemsetFn; | ||||||
560 | |||||||
561 | /// KMSAN callback for task-local function argument shadow. | ||||||
562 | StructType *MsanContextStateTy; | ||||||
563 | FunctionCallee MsanGetContextStateFn; | ||||||
564 | |||||||
565 | /// Functions for poisoning/unpoisoning local variables | ||||||
566 | FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn; | ||||||
567 | |||||||
568 | /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin | ||||||
569 | /// pointers. | ||||||
570 | FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN; | ||||||
571 | FunctionCallee MsanMetadataPtrForLoad_1_8[4]; | ||||||
572 | FunctionCallee MsanMetadataPtrForStore_1_8[4]; | ||||||
573 | FunctionCallee MsanInstrumentAsmStoreFn; | ||||||
574 | |||||||
575 | /// Helper to choose between different MsanMetadataPtrXxx(). | ||||||
576 | FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size); | ||||||
577 | |||||||
578 | /// Memory map parameters used in application-to-shadow calculation. | ||||||
579 | const MemoryMapParams *MapParams; | ||||||
580 | |||||||
581 | /// Custom memory map parameters used when -msan-shadow-base or | ||||||
582 | // -msan-origin-base is provided. | ||||||
583 | MemoryMapParams CustomMapParams; | ||||||
584 | |||||||
585 | MDNode *ColdCallWeights; | ||||||
586 | |||||||
587 | /// Branch weights for origin store. | ||||||
588 | MDNode *OriginStoreWeights; | ||||||
589 | |||||||
590 | /// An empty volatile inline asm that prevents callback merge. | ||||||
591 | InlineAsm *EmptyAsm; | ||||||
592 | }; | ||||||
593 | |||||||
594 | void insertModuleCtor(Module &M) { | ||||||
595 | getOrCreateSanitizerCtorAndInitFunctions( | ||||||
596 | M, kMsanModuleCtorName, kMsanInitName, | ||||||
597 | /*InitArgTypes=*/{}, | ||||||
598 | /*InitArgs=*/{}, | ||||||
599 | // This callback is invoked when the functions are created the first | ||||||
600 | // time. Hook them into the global ctors list in that case: | ||||||
601 | [&](Function *Ctor, FunctionCallee) { | ||||||
602 | if (!ClWithComdat) { | ||||||
603 | appendToGlobalCtors(M, Ctor, 0); | ||||||
604 | return; | ||||||
605 | } | ||||||
606 | Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName); | ||||||
607 | Ctor->setComdat(MsanCtorComdat); | ||||||
608 | appendToGlobalCtors(M, Ctor, 0, Ctor); | ||||||
609 | }); | ||||||
610 | } | ||||||
611 | |||||||
612 | /// A legacy function pass for msan instrumentation. | ||||||
613 | /// | ||||||
614 | /// Instruments functions to detect unitialized reads. | ||||||
615 | struct MemorySanitizerLegacyPass : public FunctionPass { | ||||||
616 | // Pass identification, replacement for typeid. | ||||||
617 | static char ID; | ||||||
618 | |||||||
619 | MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {}) | ||||||
620 | : FunctionPass(ID), Options(Options) {} | ||||||
621 | StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; } | ||||||
622 | |||||||
623 | void getAnalysisUsage(AnalysisUsage &AU) const override { | ||||||
624 | AU.addRequired<TargetLibraryInfoWrapperPass>(); | ||||||
625 | } | ||||||
626 | |||||||
627 | bool runOnFunction(Function &F) override { | ||||||
628 | return MSan->sanitizeFunction( | ||||||
629 | F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F)); | ||||||
630 | } | ||||||
631 | bool doInitialization(Module &M) override; | ||||||
632 | |||||||
633 | Optional<MemorySanitizer> MSan; | ||||||
634 | MemorySanitizerOptions Options; | ||||||
635 | }; | ||||||
636 | |||||||
637 | template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) { | ||||||
638 | return (Opt.getNumOccurrences() > 0) ? Opt : Default; | ||||||
639 | } | ||||||
640 | |||||||
641 | } // end anonymous namespace | ||||||
642 | |||||||
643 | MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K) | ||||||
644 | : Kernel(getOptOrDefault(ClEnableKmsan, K)), | ||||||
645 | TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)), | ||||||
646 | Recover(getOptOrDefault(ClKeepGoing, Kernel || R)) {} | ||||||
647 | |||||||
648 | PreservedAnalyses MemorySanitizerPass::run(Function &F, | ||||||
649 | FunctionAnalysisManager &FAM) { | ||||||
650 | MemorySanitizer Msan(*F.getParent(), Options); | ||||||
651 | if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F))) | ||||||
652 | return PreservedAnalyses::none(); | ||||||
653 | return PreservedAnalyses::all(); | ||||||
654 | } | ||||||
655 | |||||||
656 | PreservedAnalyses MemorySanitizerPass::run(Module &M, | ||||||
657 | ModuleAnalysisManager &AM) { | ||||||
658 | if (Options.Kernel) | ||||||
659 | return PreservedAnalyses::all(); | ||||||
660 | insertModuleCtor(M); | ||||||
661 | return PreservedAnalyses::none(); | ||||||
662 | } | ||||||
663 | |||||||
664 | char MemorySanitizerLegacyPass::ID = 0; | ||||||
665 | |||||||
666 | INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry &Registry) { | ||||||
667 | "MemorySanitizer: detects uninitialized reads.", false,static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry &Registry) { | ||||||
668 | false)static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry &Registry) { | ||||||
669 | INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry); | ||||||
670 | INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads." , "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<MemorySanitizerLegacyPass>), false, false ); Registry.registerPass(*PI, true); return PI; } static llvm ::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry & Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag , initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry )); } | ||||||
671 | "MemorySanitizer: detects uninitialized reads.", false,PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads." , "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<MemorySanitizerLegacyPass>), false, false ); Registry.registerPass(*PI, true); return PI; } static llvm ::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry & Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag , initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry )); } | ||||||
672 | false)PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads." , "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<MemorySanitizerLegacyPass>), false, false ); Registry.registerPass(*PI, true); return PI; } static llvm ::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry & Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag , initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry )); } | ||||||
673 | |||||||
674 | FunctionPass * | ||||||
675 | llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) { | ||||||
676 | return new MemorySanitizerLegacyPass(Options); | ||||||
677 | } | ||||||
678 | |||||||
679 | /// Create a non-const global initialized with the given string. | ||||||
680 | /// | ||||||
681 | /// Creates a writable global for Str so that we can pass it to the | ||||||
682 | /// run-time lib. Runtime uses first 4 bytes of the string to store the | ||||||
683 | /// frame ID, so the string needs to be mutable. | ||||||
684 | static GlobalVariable *createPrivateNonConstGlobalForString(Module &M, | ||||||
685 | StringRef Str) { | ||||||
686 | Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str); | ||||||
687 | return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false, | ||||||
688 | GlobalValue::PrivateLinkage, StrConst, ""); | ||||||
689 | } | ||||||
690 | |||||||
691 | /// Create KMSAN API callbacks. | ||||||
692 | void MemorySanitizer::createKernelApi(Module &M) { | ||||||
693 | IRBuilder<> IRB(*C); | ||||||
694 | |||||||
695 | // These will be initialized in insertKmsanPrologue(). | ||||||
696 | RetvalTLS = nullptr; | ||||||
697 | RetvalOriginTLS = nullptr; | ||||||
698 | ParamTLS = nullptr; | ||||||
699 | ParamOriginTLS = nullptr; | ||||||
700 | VAArgTLS = nullptr; | ||||||
701 | VAArgOriginTLS = nullptr; | ||||||
702 | VAArgOverflowSizeTLS = nullptr; | ||||||
703 | // OriginTLS is unused in the kernel. | ||||||
704 | OriginTLS = nullptr; | ||||||
705 | |||||||
706 | // __msan_warning() in the kernel takes an origin. | ||||||
707 | WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(), | ||||||
708 | IRB.getInt32Ty()); | ||||||
709 | // Requests the per-task context state (kmsan_context_state*) from the | ||||||
710 | // runtime library. | ||||||
711 | MsanContextStateTy = StructType::get( | ||||||
712 | ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), | ||||||
713 | ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), | ||||||
714 | ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), | ||||||
715 | ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */ | ||||||
716 | IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy, | ||||||
717 | OriginTy); | ||||||
718 | MsanGetContextStateFn = M.getOrInsertFunction( | ||||||
719 | "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0)); | ||||||
720 | |||||||
721 | Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0), | ||||||
722 | PointerType::get(IRB.getInt32Ty(), 0)); | ||||||
723 | |||||||
724 | for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) { | ||||||
725 | std::string name_load = | ||||||
726 | "__msan_metadata_ptr_for_load_" + std::to_string(size); | ||||||
727 | std::string name_store = | ||||||
728 | "__msan_metadata_ptr_for_store_" + std::to_string(size); | ||||||
729 | MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction( | ||||||
730 | name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); | ||||||
731 | MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction( | ||||||
732 | name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0)); | ||||||
733 | } | ||||||
734 | |||||||
735 | MsanMetadataPtrForLoadN = M.getOrInsertFunction( | ||||||
736 | "__msan_metadata_ptr_for_load_n", RetTy, | ||||||
737 | PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); | ||||||
738 | MsanMetadataPtrForStoreN = M.getOrInsertFunction( | ||||||
739 | "__msan_metadata_ptr_for_store_n", RetTy, | ||||||
740 | PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty()); | ||||||
741 | |||||||
742 | // Functions for poisoning and unpoisoning memory. | ||||||
743 | MsanPoisonAllocaFn = | ||||||
744 | M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(), | ||||||
745 | IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy()); | ||||||
746 | MsanUnpoisonAllocaFn = M.getOrInsertFunction( | ||||||
747 | "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy); | ||||||
748 | } | ||||||
749 | |||||||
750 | static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) { | ||||||
751 | return M.getOrInsertGlobal(Name, Ty, [&] { | ||||||
752 | return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage, | ||||||
753 | nullptr, Name, nullptr, | ||||||
754 | GlobalVariable::InitialExecTLSModel); | ||||||
755 | }); | ||||||
756 | } | ||||||
757 | |||||||
758 | /// Insert declarations for userspace-specific functions and globals. | ||||||
759 | void MemorySanitizer::createUserspaceApi(Module &M) { | ||||||
760 | IRBuilder<> IRB(*C); | ||||||
761 | // Create the callback. | ||||||
762 | // FIXME: this function should have "Cold" calling conv, | ||||||
763 | // which is not yet implemented. | ||||||
764 | StringRef WarningFnName = Recover ? "__msan_warning" | ||||||
765 | : "__msan_warning_noreturn"; | ||||||
766 | WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy()); | ||||||
767 | |||||||
768 | // Create the global TLS variables. | ||||||
769 | RetvalTLS = | ||||||
770 | getOrInsertGlobal(M, "__msan_retval_tls", | ||||||
771 | ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8)); | ||||||
772 | |||||||
773 | RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy); | ||||||
774 | |||||||
775 | ParamTLS = | ||||||
776 | getOrInsertGlobal(M, "__msan_param_tls", | ||||||
777 | ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8)); | ||||||
778 | |||||||
779 | ParamOriginTLS = | ||||||
780 | getOrInsertGlobal(M, "__msan_param_origin_tls", | ||||||
781 | ArrayType::get(OriginTy, kParamTLSSize / 4)); | ||||||
782 | |||||||
783 | VAArgTLS = | ||||||
784 | getOrInsertGlobal(M, "__msan_va_arg_tls", | ||||||
785 | ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8)); | ||||||
786 | |||||||
787 | VAArgOriginTLS = | ||||||
788 | getOrInsertGlobal(M, "__msan_va_arg_origin_tls", | ||||||
789 | ArrayType::get(OriginTy, kParamTLSSize / 4)); | ||||||
790 | |||||||
791 | VAArgOverflowSizeTLS = | ||||||
792 | getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty()); | ||||||
793 | OriginTLS = getOrInsertGlobal(M, "__msan_origin_tls", IRB.getInt32Ty()); | ||||||
794 | |||||||
795 | for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; | ||||||
796 | AccessSizeIndex++) { | ||||||
797 | unsigned AccessSize = 1 << AccessSizeIndex; | ||||||
798 | std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize); | ||||||
799 | MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction( | ||||||
800 | FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), | ||||||
801 | IRB.getInt32Ty()); | ||||||
802 | |||||||
803 | FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize); | ||||||
804 | MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction( | ||||||
805 | FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), | ||||||
806 | IRB.getInt8PtrTy(), IRB.getInt32Ty()); | ||||||
807 | } | ||||||
808 | |||||||
809 | MsanSetAllocaOrigin4Fn = M.getOrInsertFunction( | ||||||
810 | "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, | ||||||
811 | IRB.getInt8PtrTy(), IntptrTy); | ||||||
812 | MsanPoisonStackFn = | ||||||
813 | M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(), | ||||||
814 | IRB.getInt8PtrTy(), IntptrTy); | ||||||
815 | } | ||||||
816 | |||||||
817 | /// Insert extern declaration of runtime-provided functions and globals. | ||||||
818 | void MemorySanitizer::initializeCallbacks(Module &M) { | ||||||
819 | // Only do this once. | ||||||
820 | if (CallbacksInitialized) | ||||||
821 | return; | ||||||
822 | |||||||
823 | IRBuilder<> IRB(*C); | ||||||
824 | // Initialize callbacks that are common for kernel and userspace | ||||||
825 | // instrumentation. | ||||||
826 | MsanChainOriginFn = M.getOrInsertFunction( | ||||||
827 | "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty()); | ||||||
828 | MemmoveFn = M.getOrInsertFunction( | ||||||
829 | "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), | ||||||
830 | IRB.getInt8PtrTy(), IntptrTy); | ||||||
831 | MemcpyFn = M.getOrInsertFunction( | ||||||
832 | "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), | ||||||
833 | IntptrTy); | ||||||
834 | MemsetFn = M.getOrInsertFunction( | ||||||
835 | "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), | ||||||
836 | IntptrTy); | ||||||
837 | // We insert an empty inline asm after __msan_report* to avoid callback merge. | ||||||
838 | EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false), | ||||||
839 | StringRef(""), StringRef(""), | ||||||
840 | /*hasSideEffects=*/true); | ||||||
841 | |||||||
842 | MsanInstrumentAsmStoreFn = | ||||||
843 | M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(), | ||||||
844 | PointerType::get(IRB.getInt8Ty(), 0), IntptrTy); | ||||||
845 | |||||||
846 | if (CompileKernel) { | ||||||
847 | createKernelApi(M); | ||||||
848 | } else { | ||||||
849 | createUserspaceApi(M); | ||||||
850 | } | ||||||
851 | CallbacksInitialized = true; | ||||||
852 | } | ||||||
853 | |||||||
854 | FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, | ||||||
855 | int size) { | ||||||
856 | FunctionCallee *Fns = | ||||||
857 | isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8; | ||||||
858 | switch (size) { | ||||||
859 | case 1: | ||||||
860 | return Fns[0]; | ||||||
861 | case 2: | ||||||
862 | return Fns[1]; | ||||||
863 | case 4: | ||||||
864 | return Fns[2]; | ||||||
865 | case 8: | ||||||
866 | return Fns[3]; | ||||||
867 | default: | ||||||
868 | return nullptr; | ||||||
869 | } | ||||||
870 | } | ||||||
871 | |||||||
872 | /// Module-level initialization. | ||||||
873 | /// | ||||||
874 | /// inserts a call to __msan_init to the module's constructor list. | ||||||
875 | void MemorySanitizer::initializeModule(Module &M) { | ||||||
876 | auto &DL = M.getDataLayout(); | ||||||
877 | |||||||
878 | bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0; | ||||||
879 | bool OriginPassed = ClOriginBase.getNumOccurrences() > 0; | ||||||
880 | // Check the overrides first | ||||||
881 | if (ShadowPassed || OriginPassed) { | ||||||
882 | CustomMapParams.AndMask = ClAndMask; | ||||||
883 | CustomMapParams.XorMask = ClXorMask; | ||||||
884 | CustomMapParams.ShadowBase = ClShadowBase; | ||||||
885 | CustomMapParams.OriginBase = ClOriginBase; | ||||||
886 | MapParams = &CustomMapParams; | ||||||
887 | } else { | ||||||
888 | Triple TargetTriple(M.getTargetTriple()); | ||||||
889 | switch (TargetTriple.getOS()) { | ||||||
890 | case Triple::FreeBSD: | ||||||
891 | switch (TargetTriple.getArch()) { | ||||||
892 | case Triple::x86_64: | ||||||
893 | MapParams = FreeBSD_X86_MemoryMapParams.bits64; | ||||||
894 | break; | ||||||
895 | case Triple::x86: | ||||||
896 | MapParams = FreeBSD_X86_MemoryMapParams.bits32; | ||||||
897 | break; | ||||||
898 | default: | ||||||
899 | report_fatal_error("unsupported architecture"); | ||||||
900 | } | ||||||
901 | break; | ||||||
902 | case Triple::NetBSD: | ||||||
903 | switch (TargetTriple.getArch()) { | ||||||
904 | case Triple::x86_64: | ||||||
905 | MapParams = NetBSD_X86_MemoryMapParams.bits64; | ||||||
906 | break; | ||||||
907 | default: | ||||||
908 | report_fatal_error("unsupported architecture"); | ||||||
909 | } | ||||||
910 | break; | ||||||
911 | case Triple::Linux: | ||||||
912 | switch (TargetTriple.getArch()) { | ||||||
913 | case Triple::x86_64: | ||||||
914 | MapParams = Linux_X86_MemoryMapParams.bits64; | ||||||
915 | break; | ||||||
916 | case Triple::x86: | ||||||
917 | MapParams = Linux_X86_MemoryMapParams.bits32; | ||||||
918 | break; | ||||||
919 | case Triple::mips64: | ||||||
920 | case Triple::mips64el: | ||||||
921 | MapParams = Linux_MIPS_MemoryMapParams.bits64; | ||||||
922 | break; | ||||||
923 | case Triple::ppc64: | ||||||
924 | case Triple::ppc64le: | ||||||
925 | MapParams = Linux_PowerPC_MemoryMapParams.bits64; | ||||||
926 | break; | ||||||
927 | case Triple::aarch64: | ||||||
928 | case Triple::aarch64_be: | ||||||
929 | MapParams = Linux_ARM_MemoryMapParams.bits64; | ||||||
930 | break; | ||||||
931 | default: | ||||||
932 | report_fatal_error("unsupported architecture"); | ||||||
933 | } | ||||||
934 | break; | ||||||
935 | default: | ||||||
936 | report_fatal_error("unsupported operating system"); | ||||||
937 | } | ||||||
938 | } | ||||||
939 | |||||||
940 | C = &(M.getContext()); | ||||||
941 | IRBuilder<> IRB(*C); | ||||||
942 | IntptrTy = IRB.getIntPtrTy(DL); | ||||||
943 | OriginTy = IRB.getInt32Ty(); | ||||||
944 | |||||||
945 | ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000); | ||||||
946 | OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000); | ||||||
947 | |||||||
948 | if (!CompileKernel) { | ||||||
949 | if (TrackOrigins) | ||||||
950 | M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] { | ||||||
951 | return new GlobalVariable( | ||||||
952 | M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage, | ||||||
953 | IRB.getInt32(TrackOrigins), "__msan_track_origins"); | ||||||
954 | }); | ||||||
955 | |||||||
956 | if (Recover) | ||||||
957 | M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] { | ||||||
958 | return new GlobalVariable(M, IRB.getInt32Ty(), true, | ||||||
959 | GlobalValue::WeakODRLinkage, | ||||||
960 | IRB.getInt32(Recover), "__msan_keep_going"); | ||||||
961 | }); | ||||||
962 | } | ||||||
963 | } | ||||||
964 | |||||||
965 | bool MemorySanitizerLegacyPass::doInitialization(Module &M) { | ||||||
966 | if (!Options.Kernel) | ||||||
967 | insertModuleCtor(M); | ||||||
968 | MSan.emplace(M, Options); | ||||||
969 | return true; | ||||||
970 | } | ||||||
971 | |||||||
972 | namespace { | ||||||
973 | |||||||
974 | /// A helper class that handles instrumentation of VarArg | ||||||
975 | /// functions on a particular platform. | ||||||
976 | /// | ||||||
977 | /// Implementations are expected to insert the instrumentation | ||||||
978 | /// necessary to propagate argument shadow through VarArg function | ||||||
979 | /// calls. Visit* methods are called during an InstVisitor pass over | ||||||
980 | /// the function, and should avoid creating new basic blocks. A new | ||||||
981 | /// instance of this class is created for each instrumented function. | ||||||
982 | struct VarArgHelper { | ||||||
983 | virtual ~VarArgHelper() = default; | ||||||
984 | |||||||
985 | /// Visit a CallSite. | ||||||
986 | virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0; | ||||||
987 | |||||||
988 | /// Visit a va_start call. | ||||||
989 | virtual void visitVAStartInst(VAStartInst &I) = 0; | ||||||
990 | |||||||
991 | /// Visit a va_copy call. | ||||||
992 | virtual void visitVACopyInst(VACopyInst &I) = 0; | ||||||
993 | |||||||
994 | /// Finalize function instrumentation. | ||||||
995 | /// | ||||||
996 | /// This method is called after visiting all interesting (see above) | ||||||
997 | /// instructions in a function. | ||||||
998 | virtual void finalizeInstrumentation() = 0; | ||||||
999 | }; | ||||||
1000 | |||||||
1001 | struct MemorySanitizerVisitor; | ||||||
1002 | |||||||
1003 | } // end anonymous namespace | ||||||
1004 | |||||||
1005 | static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, | ||||||
1006 | MemorySanitizerVisitor &Visitor); | ||||||
1007 | |||||||
1008 | static unsigned TypeSizeToSizeIndex(unsigned TypeSize) { | ||||||
1009 | if (TypeSize <= 8) return 0; | ||||||
1010 | return Log2_32_Ceil((TypeSize + 7) / 8); | ||||||
1011 | } | ||||||
1012 | |||||||
1013 | namespace { | ||||||
1014 | |||||||
1015 | /// This class does all the work for a given function. Store and Load | ||||||
1016 | /// instructions store and load corresponding shadow and origin | ||||||
1017 | /// values. Most instructions propagate shadow from arguments to their | ||||||
1018 | /// return values. Certain instructions (most importantly, BranchInst) | ||||||
1019 | /// test their argument shadow and print reports (with a runtime call) if it's | ||||||
1020 | /// non-zero. | ||||||
1021 | struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> { | ||||||
1022 | Function &F; | ||||||
1023 | MemorySanitizer &MS; | ||||||
1024 | SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes; | ||||||
1025 | ValueMap<Value*, Value*> ShadowMap, OriginMap; | ||||||
1026 | std::unique_ptr<VarArgHelper> VAHelper; | ||||||
1027 | const TargetLibraryInfo *TLI; | ||||||
1028 | BasicBlock *ActualFnStart; | ||||||
1029 | |||||||
1030 | // The following flags disable parts of MSan instrumentation based on | ||||||
1031 | // blacklist contents and command-line options. | ||||||
1032 | bool InsertChecks; | ||||||
1033 | bool PropagateShadow; | ||||||
1034 | bool PoisonStack; | ||||||
1035 | bool PoisonUndef; | ||||||
1036 | bool CheckReturnValue; | ||||||
1037 | |||||||
1038 | struct ShadowOriginAndInsertPoint { | ||||||
1039 | Value *Shadow; | ||||||
1040 | Value *Origin; | ||||||
1041 | Instruction *OrigIns; | ||||||
1042 | |||||||
1043 | ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I) | ||||||
1044 | : Shadow(S), Origin(O), OrigIns(I) {} | ||||||
1045 | }; | ||||||
1046 | SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList; | ||||||
1047 | bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics; | ||||||
1048 | SmallSet<AllocaInst *, 16> AllocaSet; | ||||||
1049 | SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList; | ||||||
1050 | SmallVector<StoreInst *, 16> StoreList; | ||||||
1051 | |||||||
1052 | MemorySanitizerVisitor(Function &F, MemorySanitizer &MS, | ||||||
1053 | const TargetLibraryInfo &TLI) | ||||||
1054 | : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) { | ||||||
1055 | bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory); | ||||||
1056 | InsertChecks = SanitizeFunction; | ||||||
1057 | PropagateShadow = SanitizeFunction; | ||||||
1058 | PoisonStack = SanitizeFunction && ClPoisonStack; | ||||||
1059 | PoisonUndef = SanitizeFunction && ClPoisonUndef; | ||||||
1060 | // FIXME: Consider using SpecialCaseList to specify a list of functions that | ||||||
1061 | // must always return fully initialized values. For now, we hardcode "main". | ||||||
1062 | CheckReturnValue = SanitizeFunction && (F.getName() == "main"); | ||||||
1063 | |||||||
1064 | MS.initializeCallbacks(*F.getParent()); | ||||||
1065 | if (MS.CompileKernel) | ||||||
1066 | ActualFnStart = insertKmsanPrologue(F); | ||||||
1067 | else | ||||||
1068 | ActualFnStart = &F.getEntryBlock(); | ||||||
1069 | |||||||
1070 | LLVM_DEBUG(if (!InsertChecks) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '" << F.getName() << "'\n"; } } while (false) | ||||||
1071 | << "MemorySanitizer is not inserting checks into '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '" << F.getName() << "'\n"; } } while (false) | ||||||
1072 | << F.getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '" << F.getName() << "'\n"; } } while (false); | ||||||
1073 | } | ||||||
1074 | |||||||
1075 | Value *updateOrigin(Value *V, IRBuilder<> &IRB) { | ||||||
1076 | if (MS.TrackOrigins <= 1) return V; | ||||||
1077 | return IRB.CreateCall(MS.MsanChainOriginFn, V); | ||||||
1078 | } | ||||||
1079 | |||||||
1080 | Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) { | ||||||
1081 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
1082 | unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy); | ||||||
1083 | if (IntptrSize == kOriginSize) return Origin; | ||||||
1084 | assert(IntptrSize == kOriginSize * 2)((IntptrSize == kOriginSize * 2) ? static_cast<void> (0 ) : __assert_fail ("IntptrSize == kOriginSize * 2", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1084, __PRETTY_FUNCTION__)); | ||||||
1085 | Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false); | ||||||
1086 | return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8)); | ||||||
1087 | } | ||||||
1088 | |||||||
1089 | /// Fill memory range with the given origin value. | ||||||
1090 | void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr, | ||||||
1091 | unsigned Size, Align Alignment) { | ||||||
1092 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
1093 | const Align IntptrAlignment = Align(DL.getABITypeAlignment(MS.IntptrTy)); | ||||||
1094 | unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy); | ||||||
1095 | assert(IntptrAlignment >= kMinOriginAlignment)((IntptrAlignment >= kMinOriginAlignment) ? static_cast< void> (0) : __assert_fail ("IntptrAlignment >= kMinOriginAlignment" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1095, __PRETTY_FUNCTION__)); | ||||||
1096 | assert(IntptrSize >= kOriginSize)((IntptrSize >= kOriginSize) ? static_cast<void> (0) : __assert_fail ("IntptrSize >= kOriginSize", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1096, __PRETTY_FUNCTION__)); | ||||||
1097 | |||||||
1098 | unsigned Ofs = 0; | ||||||
1099 | Align CurrentAlignment = Alignment; | ||||||
1100 | if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) { | ||||||
1101 | Value *IntptrOrigin = originToIntptr(IRB, Origin); | ||||||
1102 | Value *IntptrOriginPtr = | ||||||
1103 | IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0)); | ||||||
1104 | for (unsigned i = 0; i < Size / IntptrSize; ++i) { | ||||||
1105 | Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i) | ||||||
1106 | : IntptrOriginPtr; | ||||||
1107 | IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment.value()); | ||||||
1108 | Ofs += IntptrSize / kOriginSize; | ||||||
1109 | CurrentAlignment = IntptrAlignment; | ||||||
1110 | } | ||||||
1111 | } | ||||||
1112 | |||||||
1113 | for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) { | ||||||
1114 | Value *GEP = | ||||||
1115 | i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr; | ||||||
1116 | IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment.value()); | ||||||
1117 | CurrentAlignment = kMinOriginAlignment; | ||||||
1118 | } | ||||||
1119 | } | ||||||
1120 | |||||||
1121 | void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin, | ||||||
1122 | Value *OriginPtr, Align Alignment, bool AsCall) { | ||||||
1123 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
1124 | const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment); | ||||||
1125 | unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); | ||||||
1126 | if (Shadow->getType()->isAggregateType()) { | ||||||
| |||||||
1127 | paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize, | ||||||
1128 | OriginAlignment); | ||||||
1129 | } else { | ||||||
1130 | Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); | ||||||
1131 | Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow); | ||||||
1132 | if (ConstantShadow
| ||||||
1133 | if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) | ||||||
1134 | paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize, | ||||||
1135 | OriginAlignment); | ||||||
1136 | return; | ||||||
1137 | } | ||||||
1138 | |||||||
1139 | unsigned TypeSizeInBits = | ||||||
1140 | DL.getTypeSizeInBits(ConvertedShadow->getType()); | ||||||
| |||||||
1141 | unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); | ||||||
1142 | if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { | ||||||
1143 | FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex]; | ||||||
1144 | Value *ConvertedShadow2 = IRB.CreateZExt( | ||||||
1145 | ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); | ||||||
1146 | IRB.CreateCall(Fn, {ConvertedShadow2, | ||||||
1147 | IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), | ||||||
1148 | Origin}); | ||||||
1149 | } else { | ||||||
1150 | Value *Cmp = IRB.CreateICmpNE( | ||||||
1151 | ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp"); | ||||||
1152 | Instruction *CheckTerm = SplitBlockAndInsertIfThen( | ||||||
1153 | Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights); | ||||||
1154 | IRBuilder<> IRBNew(CheckTerm); | ||||||
1155 | paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize, | ||||||
1156 | OriginAlignment); | ||||||
1157 | } | ||||||
1158 | } | ||||||
1159 | } | ||||||
1160 | |||||||
1161 | void materializeStores(bool InstrumentWithCalls) { | ||||||
1162 | for (StoreInst *SI : StoreList) { | ||||||
1163 | IRBuilder<> IRB(SI); | ||||||
1164 | Value *Val = SI->getValueOperand(); | ||||||
1165 | Value *Addr = SI->getPointerOperand(); | ||||||
1166 | Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val); | ||||||
1167 | Value *ShadowPtr, *OriginPtr; | ||||||
1168 | Type *ShadowTy = Shadow->getType(); | ||||||
1169 | const Align Alignment = assumeAligned(SI->getAlignment()); | ||||||
1170 | const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment); | ||||||
1171 | std::tie(ShadowPtr, OriginPtr) = | ||||||
1172 | getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true); | ||||||
1173 | |||||||
1174 | StoreInst *NewSI = | ||||||
1175 | IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment.value()); | ||||||
1176 | LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " STORE: " << *NewSI << "\n"; } } while (false); | ||||||
1177 | (void)NewSI; | ||||||
1178 | |||||||
1179 | if (SI->isAtomic()) | ||||||
1180 | SI->setOrdering(addReleaseOrdering(SI->getOrdering())); | ||||||
1181 | |||||||
1182 | if (MS.TrackOrigins && !SI->isAtomic()) | ||||||
1183 | storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr, | ||||||
1184 | OriginAlignment, InstrumentWithCalls); | ||||||
1185 | } | ||||||
1186 | } | ||||||
1187 | |||||||
1188 | /// Helper function to insert a warning at IRB's current insert point. | ||||||
1189 | void insertWarningFn(IRBuilder<> &IRB, Value *Origin) { | ||||||
1190 | if (!Origin) | ||||||
1191 | Origin = (Value *)IRB.getInt32(0); | ||||||
1192 | if (MS.CompileKernel) { | ||||||
1193 | IRB.CreateCall(MS.WarningFn, Origin); | ||||||
1194 | } else { | ||||||
1195 | if (MS.TrackOrigins) { | ||||||
1196 | IRB.CreateStore(Origin, MS.OriginTLS); | ||||||
1197 | } | ||||||
1198 | IRB.CreateCall(MS.WarningFn, {}); | ||||||
1199 | } | ||||||
1200 | IRB.CreateCall(MS.EmptyAsm, {}); | ||||||
1201 | // FIXME: Insert UnreachableInst if !MS.Recover? | ||||||
1202 | // This may invalidate some of the following checks and needs to be done | ||||||
1203 | // at the very end. | ||||||
1204 | } | ||||||
1205 | |||||||
1206 | void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin, | ||||||
1207 | bool AsCall) { | ||||||
1208 | IRBuilder<> IRB(OrigIns); | ||||||
1209 | LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " SHAD0 : " << *Shadow << "\n"; } } while (false); | ||||||
1210 | Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB); | ||||||
1211 | LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " SHAD1 : " << *ConvertedShadow << "\n"; } } while (false); | ||||||
1212 | |||||||
1213 | Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow); | ||||||
1214 | if (ConstantShadow) { | ||||||
1215 | if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) { | ||||||
1216 | insertWarningFn(IRB, Origin); | ||||||
1217 | } | ||||||
1218 | return; | ||||||
1219 | } | ||||||
1220 | |||||||
1221 | const DataLayout &DL = OrigIns->getModule()->getDataLayout(); | ||||||
1222 | |||||||
1223 | unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType()); | ||||||
1224 | unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits); | ||||||
1225 | if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) { | ||||||
1226 | FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex]; | ||||||
1227 | Value *ConvertedShadow2 = | ||||||
1228 | IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex))); | ||||||
1229 | IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin | ||||||
1230 | ? Origin | ||||||
1231 | : (Value *)IRB.getInt32(0)}); | ||||||
1232 | } else { | ||||||
1233 | Value *Cmp = IRB.CreateICmpNE(ConvertedShadow, | ||||||
1234 | getCleanShadow(ConvertedShadow), "_mscmp"); | ||||||
1235 | Instruction *CheckTerm = SplitBlockAndInsertIfThen( | ||||||
1236 | Cmp, OrigIns, | ||||||
1237 | /* Unreachable */ !MS.Recover, MS.ColdCallWeights); | ||||||
1238 | |||||||
1239 | IRB.SetInsertPoint(CheckTerm); | ||||||
1240 | insertWarningFn(IRB, Origin); | ||||||
1241 | LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " CHECK: " << *Cmp << "\n"; } } while (false); | ||||||
1242 | } | ||||||
1243 | } | ||||||
1244 | |||||||
1245 | void materializeChecks(bool InstrumentWithCalls) { | ||||||
1246 | for (const auto &ShadowData : InstrumentationList) { | ||||||
1247 | Instruction *OrigIns = ShadowData.OrigIns; | ||||||
1248 | Value *Shadow = ShadowData.Shadow; | ||||||
1249 | Value *Origin = ShadowData.Origin; | ||||||
1250 | materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls); | ||||||
1251 | } | ||||||
1252 | LLVM_DEBUG(dbgs() << "DONE:\n" << F)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "DONE:\n" << F; } } while ( false); | ||||||
1253 | } | ||||||
1254 | |||||||
1255 | BasicBlock *insertKmsanPrologue(Function &F) { | ||||||
1256 | BasicBlock *ret = | ||||||
1257 | SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI()); | ||||||
1258 | IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI()); | ||||||
1259 | Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {}); | ||||||
1260 | Constant *Zero = IRB.getInt32(0); | ||||||
1261 | MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1262 | {Zero, IRB.getInt32(0)}, "param_shadow"); | ||||||
1263 | MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1264 | {Zero, IRB.getInt32(1)}, "retval_shadow"); | ||||||
1265 | MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1266 | {Zero, IRB.getInt32(2)}, "va_arg_shadow"); | ||||||
1267 | MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1268 | {Zero, IRB.getInt32(3)}, "va_arg_origin"); | ||||||
1269 | MS.VAArgOverflowSizeTLS = | ||||||
1270 | IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1271 | {Zero, IRB.getInt32(4)}, "va_arg_overflow_size"); | ||||||
1272 | MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1273 | {Zero, IRB.getInt32(5)}, "param_origin"); | ||||||
1274 | MS.RetvalOriginTLS = | ||||||
1275 | IRB.CreateGEP(MS.MsanContextStateTy, ContextState, | ||||||
1276 | {Zero, IRB.getInt32(6)}, "retval_origin"); | ||||||
1277 | return ret; | ||||||
1278 | } | ||||||
1279 | |||||||
1280 | /// Add MemorySanitizer instrumentation to a function. | ||||||
1281 | bool runOnFunction() { | ||||||
1282 | // In the presence of unreachable blocks, we may see Phi nodes with | ||||||
1283 | // incoming nodes from such blocks. Since InstVisitor skips unreachable | ||||||
1284 | // blocks, such nodes will not have any shadow value associated with them. | ||||||
1285 | // It's easier to remove unreachable blocks than deal with missing shadow. | ||||||
1286 | removeUnreachableBlocks(F); | ||||||
1287 | |||||||
1288 | // Iterate all BBs in depth-first order and create shadow instructions | ||||||
1289 | // for all instructions (where applicable). | ||||||
1290 | // For PHI nodes we create dummy shadow PHIs which will be finalized later. | ||||||
1291 | for (BasicBlock *BB : depth_first(ActualFnStart)) | ||||||
1292 | visit(*BB); | ||||||
1293 | |||||||
1294 | // Finalize PHI nodes. | ||||||
1295 | for (PHINode *PN : ShadowPHINodes) { | ||||||
1296 | PHINode *PNS = cast<PHINode>(getShadow(PN)); | ||||||
1297 | PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr; | ||||||
1298 | size_t NumValues = PN->getNumIncomingValues(); | ||||||
1299 | for (size_t v = 0; v < NumValues; v++) { | ||||||
1300 | PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v)); | ||||||
1301 | if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v)); | ||||||
1302 | } | ||||||
1303 | } | ||||||
1304 | |||||||
1305 | VAHelper->finalizeInstrumentation(); | ||||||
1306 | |||||||
1307 | // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to | ||||||
1308 | // instrumenting only allocas. | ||||||
1309 | if (InstrumentLifetimeStart) { | ||||||
1310 | for (auto Item : LifetimeStartList) { | ||||||
1311 | instrumentAlloca(*Item.second, Item.first); | ||||||
1312 | AllocaSet.erase(Item.second); | ||||||
1313 | } | ||||||
1314 | } | ||||||
1315 | // Poison the allocas for which we didn't instrument the corresponding | ||||||
1316 | // lifetime intrinsics. | ||||||
1317 | for (AllocaInst *AI : AllocaSet) | ||||||
1318 | instrumentAlloca(*AI); | ||||||
1319 | |||||||
1320 | bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 && | ||||||
1321 | InstrumentationList.size() + StoreList.size() > | ||||||
1322 | (unsigned)ClInstrumentationWithCallThreshold; | ||||||
1323 | |||||||
1324 | // Insert shadow value checks. | ||||||
1325 | materializeChecks(InstrumentWithCalls); | ||||||
1326 | |||||||
1327 | // Delayed instrumentation of StoreInst. | ||||||
1328 | // This may not add new address checks. | ||||||
1329 | materializeStores(InstrumentWithCalls); | ||||||
1330 | |||||||
1331 | return true; | ||||||
1332 | } | ||||||
1333 | |||||||
1334 | /// Compute the shadow type that corresponds to a given Value. | ||||||
1335 | Type *getShadowTy(Value *V) { | ||||||
1336 | return getShadowTy(V->getType()); | ||||||
1337 | } | ||||||
1338 | |||||||
1339 | /// Compute the shadow type that corresponds to a given Type. | ||||||
1340 | Type *getShadowTy(Type *OrigTy) { | ||||||
1341 | if (!OrigTy->isSized()) { | ||||||
1342 | return nullptr; | ||||||
1343 | } | ||||||
1344 | // For integer type, shadow is the same as the original type. | ||||||
1345 | // This may return weird-sized types like i1. | ||||||
1346 | if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy)) | ||||||
1347 | return IT; | ||||||
1348 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
1349 | if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) { | ||||||
1350 | uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType()); | ||||||
1351 | return VectorType::get(IntegerType::get(*MS.C, EltSize), | ||||||
1352 | VT->getNumElements()); | ||||||
1353 | } | ||||||
1354 | if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) { | ||||||
1355 | return ArrayType::get(getShadowTy(AT->getElementType()), | ||||||
1356 | AT->getNumElements()); | ||||||
1357 | } | ||||||
1358 | if (StructType *ST = dyn_cast<StructType>(OrigTy)) { | ||||||
1359 | SmallVector<Type*, 4> Elements; | ||||||
1360 | for (unsigned i = 0, n = ST->getNumElements(); i < n; i++) | ||||||
1361 | Elements.push_back(getShadowTy(ST->getElementType(i))); | ||||||
1362 | StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked()); | ||||||
1363 | LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n"; } } while (false); | ||||||
1364 | return Res; | ||||||
1365 | } | ||||||
1366 | uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy); | ||||||
1367 | return IntegerType::get(*MS.C, TypeSize); | ||||||
1368 | } | ||||||
1369 | |||||||
1370 | /// Flatten a vector type. | ||||||
1371 | Type *getShadowTyNoVec(Type *ty) { | ||||||
1372 | if (VectorType *vt = dyn_cast<VectorType>(ty)) | ||||||
1373 | return IntegerType::get(*MS.C, vt->getBitWidth()); | ||||||
1374 | return ty; | ||||||
1375 | } | ||||||
1376 | |||||||
1377 | /// Convert a shadow value to it's flattened variant. | ||||||
1378 | Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) { | ||||||
1379 | Type *Ty = V->getType(); | ||||||
1380 | Type *NoVecTy = getShadowTyNoVec(Ty); | ||||||
1381 | if (Ty == NoVecTy) return V; | ||||||
1382 | return IRB.CreateBitCast(V, NoVecTy); | ||||||
1383 | } | ||||||
1384 | |||||||
1385 | /// Compute the integer shadow offset that corresponds to a given | ||||||
1386 | /// application address. | ||||||
1387 | /// | ||||||
1388 | /// Offset = (Addr & ~AndMask) ^ XorMask | ||||||
1389 | Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) { | ||||||
1390 | Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy); | ||||||
1391 | |||||||
1392 | uint64_t AndMask = MS.MapParams->AndMask; | ||||||
1393 | if (AndMask) | ||||||
1394 | OffsetLong = | ||||||
1395 | IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask)); | ||||||
1396 | |||||||
1397 | uint64_t XorMask = MS.MapParams->XorMask; | ||||||
1398 | if (XorMask) | ||||||
1399 | OffsetLong = | ||||||
1400 | IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask)); | ||||||
1401 | return OffsetLong; | ||||||
1402 | } | ||||||
1403 | |||||||
1404 | /// Compute the shadow and origin addresses corresponding to a given | ||||||
1405 | /// application address. | ||||||
1406 | /// | ||||||
1407 | /// Shadow = ShadowBase + Offset | ||||||
1408 | /// Origin = (OriginBase + Offset) & ~3ULL | ||||||
1409 | std::pair<Value *, Value *> | ||||||
1410 | getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy, | ||||||
1411 | MaybeAlign Alignment) { | ||||||
1412 | Value *ShadowOffset = getShadowPtrOffset(Addr, IRB); | ||||||
1413 | Value *ShadowLong = ShadowOffset; | ||||||
1414 | uint64_t ShadowBase = MS.MapParams->ShadowBase; | ||||||
1415 | if (ShadowBase != 0) { | ||||||
1416 | ShadowLong = | ||||||
1417 | IRB.CreateAdd(ShadowLong, | ||||||
1418 | ConstantInt::get(MS.IntptrTy, ShadowBase)); | ||||||
1419 | } | ||||||
1420 | Value *ShadowPtr = | ||||||
1421 | IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0)); | ||||||
1422 | Value *OriginPtr = nullptr; | ||||||
1423 | if (MS.TrackOrigins) { | ||||||
1424 | Value *OriginLong = ShadowOffset; | ||||||
1425 | uint64_t OriginBase = MS.MapParams->OriginBase; | ||||||
1426 | if (OriginBase != 0) | ||||||
1427 | OriginLong = IRB.CreateAdd(OriginLong, | ||||||
1428 | ConstantInt::get(MS.IntptrTy, OriginBase)); | ||||||
1429 | if (!Alignment || *Alignment < kMinOriginAlignment) { | ||||||
1430 | uint64_t Mask = kMinOriginAlignment.value() - 1; | ||||||
1431 | OriginLong = | ||||||
1432 | IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask)); | ||||||
1433 | } | ||||||
1434 | OriginPtr = | ||||||
1435 | IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0)); | ||||||
1436 | } | ||||||
1437 | return std::make_pair(ShadowPtr, OriginPtr); | ||||||
1438 | } | ||||||
1439 | |||||||
1440 | std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr, | ||||||
1441 | IRBuilder<> &IRB, | ||||||
1442 | Type *ShadowTy, | ||||||
1443 | bool isStore) { | ||||||
1444 | Value *ShadowOriginPtrs; | ||||||
1445 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
1446 | int Size = DL.getTypeStoreSize(ShadowTy); | ||||||
1447 | |||||||
1448 | FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size); | ||||||
1449 | Value *AddrCast = | ||||||
1450 | IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0)); | ||||||
1451 | if (Getter) { | ||||||
1452 | ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast); | ||||||
1453 | } else { | ||||||
1454 | Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); | ||||||
1455 | ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN | ||||||
1456 | : MS.MsanMetadataPtrForLoadN, | ||||||
1457 | {AddrCast, SizeVal}); | ||||||
1458 | } | ||||||
1459 | Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0); | ||||||
1460 | ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0)); | ||||||
1461 | Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1); | ||||||
1462 | |||||||
1463 | return std::make_pair(ShadowPtr, OriginPtr); | ||||||
1464 | } | ||||||
1465 | |||||||
1466 | std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB, | ||||||
1467 | Type *ShadowTy, | ||||||
1468 | MaybeAlign Alignment, | ||||||
1469 | bool isStore) { | ||||||
1470 | if (MS.CompileKernel) | ||||||
1471 | return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore); | ||||||
1472 | return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment); | ||||||
1473 | } | ||||||
1474 | |||||||
1475 | /// Compute the shadow address for a given function argument. | ||||||
1476 | /// | ||||||
1477 | /// Shadow = ParamTLS+ArgOffset. | ||||||
1478 | Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB, | ||||||
1479 | int ArgOffset) { | ||||||
1480 | Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy); | ||||||
1481 | if (ArgOffset) | ||||||
1482 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
1483 | return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0), | ||||||
1484 | "_msarg"); | ||||||
1485 | } | ||||||
1486 | |||||||
1487 | /// Compute the origin address for a given function argument. | ||||||
1488 | Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, | ||||||
1489 | int ArgOffset) { | ||||||
1490 | if (!MS.TrackOrigins) | ||||||
1491 | return nullptr; | ||||||
1492 | Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy); | ||||||
1493 | if (ArgOffset) | ||||||
1494 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
1495 | return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0), | ||||||
1496 | "_msarg_o"); | ||||||
1497 | } | ||||||
1498 | |||||||
1499 | /// Compute the shadow address for a retval. | ||||||
1500 | Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) { | ||||||
1501 | return IRB.CreatePointerCast(MS.RetvalTLS, | ||||||
1502 | PointerType::get(getShadowTy(A), 0), | ||||||
1503 | "_msret"); | ||||||
1504 | } | ||||||
1505 | |||||||
1506 | /// Compute the origin address for a retval. | ||||||
1507 | Value *getOriginPtrForRetval(IRBuilder<> &IRB) { | ||||||
1508 | // We keep a single origin for the entire retval. Might be too optimistic. | ||||||
1509 | return MS.RetvalOriginTLS; | ||||||
1510 | } | ||||||
1511 | |||||||
1512 | /// Set SV to be the shadow value for V. | ||||||
1513 | void setShadow(Value *V, Value *SV) { | ||||||
1514 | assert(!ShadowMap.count(V) && "Values may only have one shadow")((!ShadowMap.count(V) && "Values may only have one shadow" ) ? static_cast<void> (0) : __assert_fail ("!ShadowMap.count(V) && \"Values may only have one shadow\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1514, __PRETTY_FUNCTION__)); | ||||||
1515 | ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V); | ||||||
1516 | } | ||||||
1517 | |||||||
1518 | /// Set Origin to be the origin value for V. | ||||||
1519 | void setOrigin(Value *V, Value *Origin) { | ||||||
1520 | if (!MS.TrackOrigins) return; | ||||||
1521 | assert(!OriginMap.count(V) && "Values may only have one origin")((!OriginMap.count(V) && "Values may only have one origin" ) ? static_cast<void> (0) : __assert_fail ("!OriginMap.count(V) && \"Values may only have one origin\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1521, __PRETTY_FUNCTION__)); | ||||||
1522 | LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n"; } } while (false); | ||||||
1523 | OriginMap[V] = Origin; | ||||||
1524 | } | ||||||
1525 | |||||||
1526 | Constant *getCleanShadow(Type *OrigTy) { | ||||||
1527 | Type *ShadowTy = getShadowTy(OrigTy); | ||||||
1528 | if (!ShadowTy) | ||||||
1529 | return nullptr; | ||||||
1530 | return Constant::getNullValue(ShadowTy); | ||||||
1531 | } | ||||||
1532 | |||||||
1533 | /// Create a clean shadow value for a given value. | ||||||
1534 | /// | ||||||
1535 | /// Clean shadow (all zeroes) means all bits of the value are defined | ||||||
1536 | /// (initialized). | ||||||
1537 | Constant *getCleanShadow(Value *V) { | ||||||
1538 | return getCleanShadow(V->getType()); | ||||||
1539 | } | ||||||
1540 | |||||||
1541 | /// Create a dirty shadow of a given shadow type. | ||||||
1542 | Constant *getPoisonedShadow(Type *ShadowTy) { | ||||||
1543 | assert(ShadowTy)((ShadowTy) ? static_cast<void> (0) : __assert_fail ("ShadowTy" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1543, __PRETTY_FUNCTION__)); | ||||||
1544 | if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) | ||||||
1545 | return Constant::getAllOnesValue(ShadowTy); | ||||||
1546 | if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) { | ||||||
1547 | SmallVector<Constant *, 4> Vals(AT->getNumElements(), | ||||||
1548 | getPoisonedShadow(AT->getElementType())); | ||||||
1549 | return ConstantArray::get(AT, Vals); | ||||||
1550 | } | ||||||
1551 | if (StructType *ST = dyn_cast<StructType>(ShadowTy)) { | ||||||
1552 | SmallVector<Constant *, 4> Vals; | ||||||
1553 | for (unsigned i = 0, n = ST->getNumElements(); i < n; i++) | ||||||
1554 | Vals.push_back(getPoisonedShadow(ST->getElementType(i))); | ||||||
1555 | return ConstantStruct::get(ST, Vals); | ||||||
1556 | } | ||||||
1557 | llvm_unreachable("Unexpected shadow type")::llvm::llvm_unreachable_internal("Unexpected shadow type", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1557); | ||||||
1558 | } | ||||||
1559 | |||||||
1560 | /// Create a dirty shadow for a given value. | ||||||
1561 | Constant *getPoisonedShadow(Value *V) { | ||||||
1562 | Type *ShadowTy = getShadowTy(V); | ||||||
1563 | if (!ShadowTy) | ||||||
1564 | return nullptr; | ||||||
1565 | return getPoisonedShadow(ShadowTy); | ||||||
1566 | } | ||||||
1567 | |||||||
1568 | /// Create a clean (zero) origin. | ||||||
1569 | Value *getCleanOrigin() { | ||||||
1570 | return Constant::getNullValue(MS.OriginTy); | ||||||
1571 | } | ||||||
1572 | |||||||
1573 | /// Get the shadow value for a given Value. | ||||||
1574 | /// | ||||||
1575 | /// This function either returns the value set earlier with setShadow, | ||||||
1576 | /// or extracts if from ParamTLS (for function arguments). | ||||||
1577 | Value *getShadow(Value *V) { | ||||||
1578 | if (!PropagateShadow) return getCleanShadow(V); | ||||||
1579 | if (Instruction *I = dyn_cast<Instruction>(V)) { | ||||||
1580 | if (I->getMetadata("nosanitize")) | ||||||
1581 | return getCleanShadow(V); | ||||||
1582 | // For instructions the shadow is already stored in the map. | ||||||
1583 | Value *Shadow = ShadowMap[V]; | ||||||
1584 | if (!Shadow) { | ||||||
1585 | LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()); } } while (false); | ||||||
1586 | (void)I; | ||||||
1587 | assert(Shadow && "No shadow for a value")((Shadow && "No shadow for a value") ? static_cast< void> (0) : __assert_fail ("Shadow && \"No shadow for a value\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1587, __PRETTY_FUNCTION__)); | ||||||
1588 | } | ||||||
1589 | return Shadow; | ||||||
1590 | } | ||||||
1591 | if (UndefValue *U = dyn_cast<UndefValue>(V)) { | ||||||
1592 | Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V); | ||||||
1593 | LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n"; } } while (false); | ||||||
1594 | (void)U; | ||||||
1595 | return AllOnes; | ||||||
1596 | } | ||||||
1597 | if (Argument *A = dyn_cast<Argument>(V)) { | ||||||
1598 | // For arguments we compute the shadow on demand and store it in the map. | ||||||
1599 | Value **ShadowPtr = &ShadowMap[V]; | ||||||
1600 | if (*ShadowPtr) | ||||||
1601 | return *ShadowPtr; | ||||||
1602 | Function *F = A->getParent(); | ||||||
1603 | IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI()); | ||||||
1604 | unsigned ArgOffset = 0; | ||||||
1605 | const DataLayout &DL = F->getParent()->getDataLayout(); | ||||||
1606 | for (auto &FArg : F->args()) { | ||||||
1607 | if (!FArg.getType()->isSized()) { | ||||||
1608 | LLVM_DEBUG(dbgs() << "Arg is not sized\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "Arg is not sized\n"; } } while ( false); | ||||||
1609 | continue; | ||||||
1610 | } | ||||||
1611 | unsigned Size = | ||||||
1612 | FArg.hasByValAttr() | ||||||
1613 | ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType()) | ||||||
1614 | : DL.getTypeAllocSize(FArg.getType()); | ||||||
1615 | if (A == &FArg) { | ||||||
1616 | bool Overflow = ArgOffset + Size > kParamTLSSize; | ||||||
1617 | Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset); | ||||||
1618 | if (FArg.hasByValAttr()) { | ||||||
1619 | // ByVal pointer itself has clean shadow. We copy the actual | ||||||
1620 | // argument shadow to the underlying memory. | ||||||
1621 | // Figure out maximal valid memcpy alignment. | ||||||
1622 | const Align ArgAlign = DL.getValueOrABITypeAlignment( | ||||||
1623 | MaybeAlign(FArg.getParamAlignment()), | ||||||
1624 | A->getType()->getPointerElementType()); | ||||||
1625 | Value *CpShadowPtr = | ||||||
1626 | getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign, | ||||||
1627 | /*isStore*/ true) | ||||||
1628 | .first; | ||||||
1629 | // TODO(glider): need to copy origins. | ||||||
1630 | if (Overflow) { | ||||||
1631 | // ParamTLS overflow. | ||||||
1632 | EntryIRB.CreateMemSet( | ||||||
1633 | CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()), | ||||||
1634 | Size, ArgAlign); | ||||||
1635 | } else { | ||||||
1636 | const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment); | ||||||
1637 | Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base, | ||||||
1638 | CopyAlign, Size); | ||||||
1639 | LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " ByValCpy: " << *Cpy << "\n"; } } while (false); | ||||||
1640 | (void)Cpy; | ||||||
1641 | } | ||||||
1642 | *ShadowPtr = getCleanShadow(V); | ||||||
1643 | } else { | ||||||
1644 | if (Overflow) { | ||||||
1645 | // ParamTLS overflow. | ||||||
1646 | *ShadowPtr = getCleanShadow(V); | ||||||
1647 | } else { | ||||||
1648 | *ShadowPtr = EntryIRB.CreateAlignedLoad( | ||||||
1649 | getShadowTy(&FArg), Base, kShadowTLSAlignment.value()); | ||||||
1650 | } | ||||||
1651 | } | ||||||
1652 | LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n"; } } while (false ) | ||||||
1653 | << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n"; } } while (false ); | ||||||
1654 | if (MS.TrackOrigins && !Overflow) { | ||||||
1655 | Value *OriginPtr = | ||||||
1656 | getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset); | ||||||
1657 | setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr)); | ||||||
1658 | } else { | ||||||
1659 | setOrigin(A, getCleanOrigin()); | ||||||
1660 | } | ||||||
1661 | } | ||||||
1662 | ArgOffset += alignTo(Size, kShadowTLSAlignment); | ||||||
1663 | } | ||||||
1664 | assert(*ShadowPtr && "Could not find shadow for an argument")((*ShadowPtr && "Could not find shadow for an argument" ) ? static_cast<void> (0) : __assert_fail ("*ShadowPtr && \"Could not find shadow for an argument\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1664, __PRETTY_FUNCTION__)); | ||||||
1665 | return *ShadowPtr; | ||||||
1666 | } | ||||||
1667 | // For everything else the shadow is zero. | ||||||
1668 | return getCleanShadow(V); | ||||||
1669 | } | ||||||
1670 | |||||||
1671 | /// Get the shadow for i-th argument of the instruction I. | ||||||
1672 | Value *getShadow(Instruction *I, int i) { | ||||||
1673 | return getShadow(I->getOperand(i)); | ||||||
1674 | } | ||||||
1675 | |||||||
1676 | /// Get the origin for a value. | ||||||
1677 | Value *getOrigin(Value *V) { | ||||||
1678 | if (!MS.TrackOrigins) return nullptr; | ||||||
1679 | if (!PropagateShadow) return getCleanOrigin(); | ||||||
1680 | if (isa<Constant>(V)) return getCleanOrigin(); | ||||||
1681 | assert((isa<Instruction>(V) || isa<Argument>(V)) &&(((isa<Instruction>(V) || isa<Argument>(V)) && "Unexpected value type in getOrigin()") ? static_cast<void > (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1682, __PRETTY_FUNCTION__)) | ||||||
1682 | "Unexpected value type in getOrigin()")(((isa<Instruction>(V) || isa<Argument>(V)) && "Unexpected value type in getOrigin()") ? static_cast<void > (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1682, __PRETTY_FUNCTION__)); | ||||||
1683 | if (Instruction *I = dyn_cast<Instruction>(V)) { | ||||||
1684 | if (I->getMetadata("nosanitize")) | ||||||
1685 | return getCleanOrigin(); | ||||||
1686 | } | ||||||
1687 | Value *Origin = OriginMap[V]; | ||||||
1688 | assert(Origin && "Missing origin")((Origin && "Missing origin") ? static_cast<void> (0) : __assert_fail ("Origin && \"Missing origin\"", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1688, __PRETTY_FUNCTION__)); | ||||||
1689 | return Origin; | ||||||
1690 | } | ||||||
1691 | |||||||
1692 | /// Get the origin for i-th argument of the instruction I. | ||||||
1693 | Value *getOrigin(Instruction *I, int i) { | ||||||
1694 | return getOrigin(I->getOperand(i)); | ||||||
1695 | } | ||||||
1696 | |||||||
1697 | /// Remember the place where a shadow check should be inserted. | ||||||
1698 | /// | ||||||
1699 | /// This location will be later instrumented with a check that will print a | ||||||
1700 | /// UMR warning in runtime if the shadow value is not 0. | ||||||
1701 | void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) { | ||||||
1702 | assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1702, __PRETTY_FUNCTION__)); | ||||||
1703 | if (!InsertChecks) return; | ||||||
1704 | #ifndef NDEBUG | ||||||
1705 | Type *ShadowTy = Shadow->getType(); | ||||||
1706 | assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&(((isa<IntegerType>(ShadowTy) || isa<VectorType>( ShadowTy)) && "Can only insert checks for integer and vector shadow types" ) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1707, __PRETTY_FUNCTION__)) | ||||||
1707 | "Can only insert checks for integer and vector shadow types")(((isa<IntegerType>(ShadowTy) || isa<VectorType>( ShadowTy)) && "Can only insert checks for integer and vector shadow types" ) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1707, __PRETTY_FUNCTION__)); | ||||||
1708 | #endif | ||||||
1709 | InstrumentationList.push_back( | ||||||
1710 | ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns)); | ||||||
1711 | } | ||||||
1712 | |||||||
1713 | /// Remember the place where a shadow check should be inserted. | ||||||
1714 | /// | ||||||
1715 | /// This location will be later instrumented with a check that will print a | ||||||
1716 | /// UMR warning in runtime if the value is not fully defined. | ||||||
1717 | void insertShadowCheck(Value *Val, Instruction *OrigIns) { | ||||||
1718 | assert(Val)((Val) ? static_cast<void> (0) : __assert_fail ("Val", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1718, __PRETTY_FUNCTION__)); | ||||||
1719 | Value *Shadow, *Origin; | ||||||
1720 | if (ClCheckConstantShadow) { | ||||||
1721 | Shadow = getShadow(Val); | ||||||
1722 | if (!Shadow) return; | ||||||
1723 | Origin = getOrigin(Val); | ||||||
1724 | } else { | ||||||
1725 | Shadow = dyn_cast_or_null<Instruction>(getShadow(Val)); | ||||||
1726 | if (!Shadow) return; | ||||||
1727 | Origin = dyn_cast_or_null<Instruction>(getOrigin(Val)); | ||||||
1728 | } | ||||||
1729 | insertShadowCheck(Shadow, Origin, OrigIns); | ||||||
1730 | } | ||||||
1731 | |||||||
1732 | AtomicOrdering addReleaseOrdering(AtomicOrdering a) { | ||||||
1733 | switch (a) { | ||||||
1734 | case AtomicOrdering::NotAtomic: | ||||||
1735 | return AtomicOrdering::NotAtomic; | ||||||
1736 | case AtomicOrdering::Unordered: | ||||||
1737 | case AtomicOrdering::Monotonic: | ||||||
1738 | case AtomicOrdering::Release: | ||||||
1739 | return AtomicOrdering::Release; | ||||||
1740 | case AtomicOrdering::Acquire: | ||||||
1741 | case AtomicOrdering::AcquireRelease: | ||||||
1742 | return AtomicOrdering::AcquireRelease; | ||||||
1743 | case AtomicOrdering::SequentiallyConsistent: | ||||||
1744 | return AtomicOrdering::SequentiallyConsistent; | ||||||
1745 | } | ||||||
1746 | llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1746); | ||||||
1747 | } | ||||||
1748 | |||||||
1749 | AtomicOrdering addAcquireOrdering(AtomicOrdering a) { | ||||||
1750 | switch (a) { | ||||||
1751 | case AtomicOrdering::NotAtomic: | ||||||
1752 | return AtomicOrdering::NotAtomic; | ||||||
1753 | case AtomicOrdering::Unordered: | ||||||
1754 | case AtomicOrdering::Monotonic: | ||||||
1755 | case AtomicOrdering::Acquire: | ||||||
1756 | return AtomicOrdering::Acquire; | ||||||
1757 | case AtomicOrdering::Release: | ||||||
1758 | case AtomicOrdering::AcquireRelease: | ||||||
1759 | return AtomicOrdering::AcquireRelease; | ||||||
1760 | case AtomicOrdering::SequentiallyConsistent: | ||||||
1761 | return AtomicOrdering::SequentiallyConsistent; | ||||||
1762 | } | ||||||
1763 | llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1763); | ||||||
1764 | } | ||||||
1765 | |||||||
1766 | // ------------------- Visitors. | ||||||
1767 | using InstVisitor<MemorySanitizerVisitor>::visit; | ||||||
1768 | void visit(Instruction &I) { | ||||||
1769 | if (!I.getMetadata("nosanitize")) | ||||||
1770 | InstVisitor<MemorySanitizerVisitor>::visit(I); | ||||||
1771 | } | ||||||
1772 | |||||||
1773 | /// Instrument LoadInst | ||||||
1774 | /// | ||||||
1775 | /// Loads the corresponding shadow and (optionally) origin. | ||||||
1776 | /// Optionally, checks that the load address is fully defined. | ||||||
1777 | void visitLoadInst(LoadInst &I) { | ||||||
1778 | assert(I.getType()->isSized() && "Load type must have size")((I.getType()->isSized() && "Load type must have size" ) ? static_cast<void> (0) : __assert_fail ("I.getType()->isSized() && \"Load type must have size\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1778, __PRETTY_FUNCTION__)); | ||||||
1779 | assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0) : __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1779, __PRETTY_FUNCTION__)); | ||||||
1780 | IRBuilder<> IRB(I.getNextNode()); | ||||||
1781 | Type *ShadowTy = getShadowTy(&I); | ||||||
1782 | Value *Addr = I.getPointerOperand(); | ||||||
1783 | Value *ShadowPtr = nullptr, *OriginPtr = nullptr; | ||||||
1784 | const Align Alignment = assumeAligned(I.getAlignment()); | ||||||
1785 | if (PropagateShadow) { | ||||||
1786 | std::tie(ShadowPtr, OriginPtr) = | ||||||
1787 | getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); | ||||||
1788 | setShadow(&I, IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, | ||||||
1789 | Alignment.value(), "_msld")); | ||||||
1790 | } else { | ||||||
1791 | setShadow(&I, getCleanShadow(&I)); | ||||||
1792 | } | ||||||
1793 | |||||||
1794 | if (ClCheckAccessAddress) | ||||||
1795 | insertShadowCheck(I.getPointerOperand(), &I); | ||||||
1796 | |||||||
1797 | if (I.isAtomic()) | ||||||
1798 | I.setOrdering(addAcquireOrdering(I.getOrdering())); | ||||||
1799 | |||||||
1800 | if (MS.TrackOrigins) { | ||||||
1801 | if (PropagateShadow) { | ||||||
1802 | const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment); | ||||||
1803 | setOrigin(&I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, | ||||||
1804 | OriginAlignment.value())); | ||||||
1805 | } else { | ||||||
1806 | setOrigin(&I, getCleanOrigin()); | ||||||
1807 | } | ||||||
1808 | } | ||||||
1809 | } | ||||||
1810 | |||||||
1811 | /// Instrument StoreInst | ||||||
1812 | /// | ||||||
1813 | /// Stores the corresponding shadow and (optionally) origin. | ||||||
1814 | /// Optionally, checks that the store address is fully defined. | ||||||
1815 | void visitStoreInst(StoreInst &I) { | ||||||
1816 | StoreList.push_back(&I); | ||||||
1817 | if (ClCheckAccessAddress) | ||||||
1818 | insertShadowCheck(I.getPointerOperand(), &I); | ||||||
1819 | } | ||||||
1820 | |||||||
1821 | void handleCASOrRMW(Instruction &I) { | ||||||
1822 | assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I))((isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst> (I)) ? static_cast<void> (0) : __assert_fail ("isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 1822, __PRETTY_FUNCTION__)); | ||||||
1823 | |||||||
1824 | IRBuilder<> IRB(&I); | ||||||
1825 | Value *Addr = I.getOperand(0); | ||||||
1826 | Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(), Align::None(), | ||||||
1827 | /*isStore*/ true) | ||||||
1828 | .first; | ||||||
1829 | |||||||
1830 | if (ClCheckAccessAddress) | ||||||
1831 | insertShadowCheck(Addr, &I); | ||||||
1832 | |||||||
1833 | // Only test the conditional argument of cmpxchg instruction. | ||||||
1834 | // The other argument can potentially be uninitialized, but we can not | ||||||
1835 | // detect this situation reliably without possible false positives. | ||||||
1836 | if (isa<AtomicCmpXchgInst>(I)) | ||||||
1837 | insertShadowCheck(I.getOperand(1), &I); | ||||||
1838 | |||||||
1839 | IRB.CreateStore(getCleanShadow(&I), ShadowPtr); | ||||||
1840 | |||||||
1841 | setShadow(&I, getCleanShadow(&I)); | ||||||
1842 | setOrigin(&I, getCleanOrigin()); | ||||||
1843 | } | ||||||
1844 | |||||||
1845 | void visitAtomicRMWInst(AtomicRMWInst &I) { | ||||||
1846 | handleCASOrRMW(I); | ||||||
1847 | I.setOrdering(addReleaseOrdering(I.getOrdering())); | ||||||
1848 | } | ||||||
1849 | |||||||
1850 | void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { | ||||||
1851 | handleCASOrRMW(I); | ||||||
1852 | I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering())); | ||||||
1853 | } | ||||||
1854 | |||||||
1855 | // Vector manipulation. | ||||||
1856 | void visitExtractElementInst(ExtractElementInst &I) { | ||||||
1857 | insertShadowCheck(I.getOperand(1), &I); | ||||||
1858 | IRBuilder<> IRB(&I); | ||||||
1859 | setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1), | ||||||
1860 | "_msprop")); | ||||||
1861 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1862 | } | ||||||
1863 | |||||||
1864 | void visitInsertElementInst(InsertElementInst &I) { | ||||||
1865 | insertShadowCheck(I.getOperand(2), &I); | ||||||
1866 | IRBuilder<> IRB(&I); | ||||||
1867 | setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1), | ||||||
1868 | I.getOperand(2), "_msprop")); | ||||||
1869 | setOriginForNaryOp(I); | ||||||
1870 | } | ||||||
1871 | |||||||
1872 | void visitShuffleVectorInst(ShuffleVectorInst &I) { | ||||||
1873 | insertShadowCheck(I.getOperand(2), &I); | ||||||
1874 | IRBuilder<> IRB(&I); | ||||||
1875 | setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1), | ||||||
1876 | I.getOperand(2), "_msprop")); | ||||||
1877 | setOriginForNaryOp(I); | ||||||
1878 | } | ||||||
1879 | |||||||
1880 | // Casts. | ||||||
1881 | void visitSExtInst(SExtInst &I) { | ||||||
1882 | IRBuilder<> IRB(&I); | ||||||
1883 | setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop")); | ||||||
1884 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1885 | } | ||||||
1886 | |||||||
1887 | void visitZExtInst(ZExtInst &I) { | ||||||
1888 | IRBuilder<> IRB(&I); | ||||||
1889 | setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop")); | ||||||
1890 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1891 | } | ||||||
1892 | |||||||
1893 | void visitTruncInst(TruncInst &I) { | ||||||
1894 | IRBuilder<> IRB(&I); | ||||||
1895 | setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop")); | ||||||
1896 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1897 | } | ||||||
1898 | |||||||
1899 | void visitBitCastInst(BitCastInst &I) { | ||||||
1900 | // Special case: if this is the bitcast (there is exactly 1 allowed) between | ||||||
1901 | // a musttail call and a ret, don't instrument. New instructions are not | ||||||
1902 | // allowed after a musttail call. | ||||||
1903 | if (auto *CI = dyn_cast<CallInst>(I.getOperand(0))) | ||||||
1904 | if (CI->isMustTailCall()) | ||||||
1905 | return; | ||||||
1906 | IRBuilder<> IRB(&I); | ||||||
1907 | setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I))); | ||||||
1908 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1909 | } | ||||||
1910 | |||||||
1911 | void visitPtrToIntInst(PtrToIntInst &I) { | ||||||
1912 | IRBuilder<> IRB(&I); | ||||||
1913 | setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false, | ||||||
1914 | "_msprop_ptrtoint")); | ||||||
1915 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1916 | } | ||||||
1917 | |||||||
1918 | void visitIntToPtrInst(IntToPtrInst &I) { | ||||||
1919 | IRBuilder<> IRB(&I); | ||||||
1920 | setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false, | ||||||
1921 | "_msprop_inttoptr")); | ||||||
1922 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
1923 | } | ||||||
1924 | |||||||
1925 | void visitFPToSIInst(CastInst& I) { handleShadowOr(I); } | ||||||
1926 | void visitFPToUIInst(CastInst& I) { handleShadowOr(I); } | ||||||
1927 | void visitSIToFPInst(CastInst& I) { handleShadowOr(I); } | ||||||
1928 | void visitUIToFPInst(CastInst& I) { handleShadowOr(I); } | ||||||
1929 | void visitFPExtInst(CastInst& I) { handleShadowOr(I); } | ||||||
1930 | void visitFPTruncInst(CastInst& I) { handleShadowOr(I); } | ||||||
1931 | |||||||
1932 | /// Propagate shadow for bitwise AND. | ||||||
1933 | /// | ||||||
1934 | /// This code is exact, i.e. if, for example, a bit in the left argument | ||||||
1935 | /// is defined and 0, then neither the value not definedness of the | ||||||
1936 | /// corresponding bit in B don't affect the resulting shadow. | ||||||
1937 | void visitAnd(BinaryOperator &I) { | ||||||
1938 | IRBuilder<> IRB(&I); | ||||||
1939 | // "And" of 0 and a poisoned value results in unpoisoned value. | ||||||
1940 | // 1&1 => 1; 0&1 => 0; p&1 => p; | ||||||
1941 | // 1&0 => 0; 0&0 => 0; p&0 => 0; | ||||||
1942 | // 1&p => p; 0&p => 0; p&p => p; | ||||||
1943 | // S = (S1 & S2) | (V1 & S2) | (S1 & V2) | ||||||
1944 | Value *S1 = getShadow(&I, 0); | ||||||
1945 | Value *S2 = getShadow(&I, 1); | ||||||
1946 | Value *V1 = I.getOperand(0); | ||||||
1947 | Value *V2 = I.getOperand(1); | ||||||
1948 | if (V1->getType() != S1->getType()) { | ||||||
1949 | V1 = IRB.CreateIntCast(V1, S1->getType(), false); | ||||||
1950 | V2 = IRB.CreateIntCast(V2, S2->getType(), false); | ||||||
1951 | } | ||||||
1952 | Value *S1S2 = IRB.CreateAnd(S1, S2); | ||||||
1953 | Value *V1S2 = IRB.CreateAnd(V1, S2); | ||||||
1954 | Value *S1V2 = IRB.CreateAnd(S1, V2); | ||||||
1955 | setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2})); | ||||||
1956 | setOriginForNaryOp(I); | ||||||
1957 | } | ||||||
1958 | |||||||
1959 | void visitOr(BinaryOperator &I) { | ||||||
1960 | IRBuilder<> IRB(&I); | ||||||
1961 | // "Or" of 1 and a poisoned value results in unpoisoned value. | ||||||
1962 | // 1|1 => 1; 0|1 => 1; p|1 => 1; | ||||||
1963 | // 1|0 => 1; 0|0 => 0; p|0 => p; | ||||||
1964 | // 1|p => 1; 0|p => p; p|p => p; | ||||||
1965 | // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2) | ||||||
1966 | Value *S1 = getShadow(&I, 0); | ||||||
1967 | Value *S2 = getShadow(&I, 1); | ||||||
1968 | Value *V1 = IRB.CreateNot(I.getOperand(0)); | ||||||
1969 | Value *V2 = IRB.CreateNot(I.getOperand(1)); | ||||||
1970 | if (V1->getType() != S1->getType()) { | ||||||
1971 | V1 = IRB.CreateIntCast(V1, S1->getType(), false); | ||||||
1972 | V2 = IRB.CreateIntCast(V2, S2->getType(), false); | ||||||
1973 | } | ||||||
1974 | Value *S1S2 = IRB.CreateAnd(S1, S2); | ||||||
1975 | Value *V1S2 = IRB.CreateAnd(V1, S2); | ||||||
1976 | Value *S1V2 = IRB.CreateAnd(S1, V2); | ||||||
1977 | setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2})); | ||||||
1978 | setOriginForNaryOp(I); | ||||||
1979 | } | ||||||
1980 | |||||||
1981 | /// Default propagation of shadow and/or origin. | ||||||
1982 | /// | ||||||
1983 | /// This class implements the general case of shadow propagation, used in all | ||||||
1984 | /// cases where we don't know and/or don't care about what the operation | ||||||
1985 | /// actually does. It converts all input shadow values to a common type | ||||||
1986 | /// (extending or truncating as necessary), and bitwise OR's them. | ||||||
1987 | /// | ||||||
1988 | /// This is much cheaper than inserting checks (i.e. requiring inputs to be | ||||||
1989 | /// fully initialized), and less prone to false positives. | ||||||
1990 | /// | ||||||
1991 | /// This class also implements the general case of origin propagation. For a | ||||||
1992 | /// Nary operation, result origin is set to the origin of an argument that is | ||||||
1993 | /// not entirely initialized. If there is more than one such arguments, the | ||||||
1994 | /// rightmost of them is picked. It does not matter which one is picked if all | ||||||
1995 | /// arguments are initialized. | ||||||
1996 | template <bool CombineShadow> | ||||||
1997 | class Combiner { | ||||||
1998 | Value *Shadow = nullptr; | ||||||
1999 | Value *Origin = nullptr; | ||||||
2000 | IRBuilder<> &IRB; | ||||||
2001 | MemorySanitizerVisitor *MSV; | ||||||
2002 | |||||||
2003 | public: | ||||||
2004 | Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB) | ||||||
2005 | : IRB(IRB), MSV(MSV) {} | ||||||
2006 | |||||||
2007 | /// Add a pair of shadow and origin values to the mix. | ||||||
2008 | Combiner &Add(Value *OpShadow, Value *OpOrigin) { | ||||||
2009 | if (CombineShadow) { | ||||||
2010 | assert(OpShadow)((OpShadow) ? static_cast<void> (0) : __assert_fail ("OpShadow" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2010, __PRETTY_FUNCTION__)); | ||||||
2011 | if (!Shadow) | ||||||
2012 | Shadow = OpShadow; | ||||||
2013 | else { | ||||||
2014 | OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType()); | ||||||
2015 | Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop"); | ||||||
2016 | } | ||||||
2017 | } | ||||||
2018 | |||||||
2019 | if (MSV->MS.TrackOrigins) { | ||||||
2020 | assert(OpOrigin)((OpOrigin) ? static_cast<void> (0) : __assert_fail ("OpOrigin" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2020, __PRETTY_FUNCTION__)); | ||||||
2021 | if (!Origin) { | ||||||
2022 | Origin = OpOrigin; | ||||||
2023 | } else { | ||||||
2024 | Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin); | ||||||
2025 | // No point in adding something that might result in 0 origin value. | ||||||
2026 | if (!ConstOrigin || !ConstOrigin->isNullValue()) { | ||||||
2027 | Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB); | ||||||
2028 | Value *Cond = | ||||||
2029 | IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow)); | ||||||
2030 | Origin = IRB.CreateSelect(Cond, OpOrigin, Origin); | ||||||
2031 | } | ||||||
2032 | } | ||||||
2033 | } | ||||||
2034 | return *this; | ||||||
2035 | } | ||||||
2036 | |||||||
2037 | /// Add an application value to the mix. | ||||||
2038 | Combiner &Add(Value *V) { | ||||||
2039 | Value *OpShadow = MSV->getShadow(V); | ||||||
2040 | Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr; | ||||||
2041 | return Add(OpShadow, OpOrigin); | ||||||
2042 | } | ||||||
2043 | |||||||
2044 | /// Set the current combined values as the given instruction's shadow | ||||||
2045 | /// and origin. | ||||||
2046 | void Done(Instruction *I) { | ||||||
2047 | if (CombineShadow) { | ||||||
2048 | assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2048, __PRETTY_FUNCTION__)); | ||||||
2049 | Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I)); | ||||||
2050 | MSV->setShadow(I, Shadow); | ||||||
2051 | } | ||||||
2052 | if (MSV->MS.TrackOrigins) { | ||||||
2053 | assert(Origin)((Origin) ? static_cast<void> (0) : __assert_fail ("Origin" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2053, __PRETTY_FUNCTION__)); | ||||||
2054 | MSV->setOrigin(I, Origin); | ||||||
2055 | } | ||||||
2056 | } | ||||||
2057 | }; | ||||||
2058 | |||||||
2059 | using ShadowAndOriginCombiner = Combiner<true>; | ||||||
2060 | using OriginCombiner = Combiner<false>; | ||||||
2061 | |||||||
2062 | /// Propagate origin for arbitrary operation. | ||||||
2063 | void setOriginForNaryOp(Instruction &I) { | ||||||
2064 | if (!MS.TrackOrigins) return; | ||||||
2065 | IRBuilder<> IRB(&I); | ||||||
2066 | OriginCombiner OC(this, IRB); | ||||||
2067 | for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI) | ||||||
2068 | OC.Add(OI->get()); | ||||||
2069 | OC.Done(&I); | ||||||
2070 | } | ||||||
2071 | |||||||
2072 | size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) { | ||||||
2073 | assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&((!(Ty->isVectorTy() && Ty->getScalarType()-> isPointerTy()) && "Vector of pointers is not a valid shadow type" ) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2074, __PRETTY_FUNCTION__)) | ||||||
2074 | "Vector of pointers is not a valid shadow type")((!(Ty->isVectorTy() && Ty->getScalarType()-> isPointerTy()) && "Vector of pointers is not a valid shadow type" ) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2074, __PRETTY_FUNCTION__)); | ||||||
2075 | return Ty->isVectorTy() ? | ||||||
2076 | Ty->getVectorNumElements() * Ty->getScalarSizeInBits() : | ||||||
2077 | Ty->getPrimitiveSizeInBits(); | ||||||
2078 | } | ||||||
2079 | |||||||
2080 | /// Cast between two shadow types, extending or truncating as | ||||||
2081 | /// necessary. | ||||||
2082 | Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy, | ||||||
2083 | bool Signed = false) { | ||||||
2084 | Type *srcTy = V->getType(); | ||||||
2085 | size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy); | ||||||
2086 | size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy); | ||||||
2087 | if (srcSizeInBits > 1 && dstSizeInBits == 1) | ||||||
2088 | return IRB.CreateICmpNE(V, getCleanShadow(V)); | ||||||
2089 | |||||||
2090 | if (dstTy->isIntegerTy() && srcTy->isIntegerTy()) | ||||||
2091 | return IRB.CreateIntCast(V, dstTy, Signed); | ||||||
2092 | if (dstTy->isVectorTy() && srcTy->isVectorTy() && | ||||||
2093 | dstTy->getVectorNumElements() == srcTy->getVectorNumElements()) | ||||||
2094 | return IRB.CreateIntCast(V, dstTy, Signed); | ||||||
2095 | Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits)); | ||||||
2096 | Value *V2 = | ||||||
2097 | IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed); | ||||||
2098 | return IRB.CreateBitCast(V2, dstTy); | ||||||
2099 | // TODO: handle struct types. | ||||||
2100 | } | ||||||
2101 | |||||||
2102 | /// Cast an application value to the type of its own shadow. | ||||||
2103 | Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) { | ||||||
2104 | Type *ShadowTy = getShadowTy(V); | ||||||
2105 | if (V->getType() == ShadowTy) | ||||||
2106 | return V; | ||||||
2107 | if (V->getType()->isPtrOrPtrVectorTy()) | ||||||
2108 | return IRB.CreatePtrToInt(V, ShadowTy); | ||||||
2109 | else | ||||||
2110 | return IRB.CreateBitCast(V, ShadowTy); | ||||||
2111 | } | ||||||
2112 | |||||||
2113 | /// Propagate shadow for arbitrary operation. | ||||||
2114 | void handleShadowOr(Instruction &I) { | ||||||
2115 | IRBuilder<> IRB(&I); | ||||||
2116 | ShadowAndOriginCombiner SC(this, IRB); | ||||||
2117 | for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI) | ||||||
2118 | SC.Add(OI->get()); | ||||||
2119 | SC.Done(&I); | ||||||
2120 | } | ||||||
2121 | |||||||
2122 | void visitFNeg(UnaryOperator &I) { handleShadowOr(I); } | ||||||
2123 | |||||||
2124 | // Handle multiplication by constant. | ||||||
2125 | // | ||||||
2126 | // Handle a special case of multiplication by constant that may have one or | ||||||
2127 | // more zeros in the lower bits. This makes corresponding number of lower bits | ||||||
2128 | // of the result zero as well. We model it by shifting the other operand | ||||||
2129 | // shadow left by the required number of bits. Effectively, we transform | ||||||
2130 | // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B). | ||||||
2131 | // We use multiplication by 2**N instead of shift to cover the case of | ||||||
2132 | // multiplication by 0, which may occur in some elements of a vector operand. | ||||||
2133 | void handleMulByConstant(BinaryOperator &I, Constant *ConstArg, | ||||||
2134 | Value *OtherArg) { | ||||||
2135 | Constant *ShadowMul; | ||||||
2136 | Type *Ty = ConstArg->getType(); | ||||||
2137 | if (Ty->isVectorTy()) { | ||||||
2138 | unsigned NumElements = Ty->getVectorNumElements(); | ||||||
2139 | Type *EltTy = Ty->getSequentialElementType(); | ||||||
2140 | SmallVector<Constant *, 16> Elements; | ||||||
2141 | for (unsigned Idx = 0; Idx < NumElements; ++Idx) { | ||||||
2142 | if (ConstantInt *Elt = | ||||||
2143 | dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) { | ||||||
2144 | const APInt &V = Elt->getValue(); | ||||||
2145 | APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros(); | ||||||
2146 | Elements.push_back(ConstantInt::get(EltTy, V2)); | ||||||
2147 | } else { | ||||||
2148 | Elements.push_back(ConstantInt::get(EltTy, 1)); | ||||||
2149 | } | ||||||
2150 | } | ||||||
2151 | ShadowMul = ConstantVector::get(Elements); | ||||||
2152 | } else { | ||||||
2153 | if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) { | ||||||
2154 | const APInt &V = Elt->getValue(); | ||||||
2155 | APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros(); | ||||||
2156 | ShadowMul = ConstantInt::get(Ty, V2); | ||||||
2157 | } else { | ||||||
2158 | ShadowMul = ConstantInt::get(Ty, 1); | ||||||
2159 | } | ||||||
2160 | } | ||||||
2161 | |||||||
2162 | IRBuilder<> IRB(&I); | ||||||
2163 | setShadow(&I, | ||||||
2164 | IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst")); | ||||||
2165 | setOrigin(&I, getOrigin(OtherArg)); | ||||||
2166 | } | ||||||
2167 | |||||||
2168 | void visitMul(BinaryOperator &I) { | ||||||
2169 | Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0)); | ||||||
2170 | Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1)); | ||||||
2171 | if (constOp0 && !constOp1) | ||||||
2172 | handleMulByConstant(I, constOp0, I.getOperand(1)); | ||||||
2173 | else if (constOp1 && !constOp0) | ||||||
2174 | handleMulByConstant(I, constOp1, I.getOperand(0)); | ||||||
2175 | else | ||||||
2176 | handleShadowOr(I); | ||||||
2177 | } | ||||||
2178 | |||||||
2179 | void visitFAdd(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2180 | void visitFSub(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2181 | void visitFMul(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2182 | void visitAdd(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2183 | void visitSub(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2184 | void visitXor(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2185 | |||||||
2186 | void handleIntegerDiv(Instruction &I) { | ||||||
2187 | IRBuilder<> IRB(&I); | ||||||
2188 | // Strict on the second argument. | ||||||
2189 | insertShadowCheck(I.getOperand(1), &I); | ||||||
2190 | setShadow(&I, getShadow(&I, 0)); | ||||||
2191 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
2192 | } | ||||||
2193 | |||||||
2194 | void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); } | ||||||
2195 | void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); } | ||||||
2196 | void visitURem(BinaryOperator &I) { handleIntegerDiv(I); } | ||||||
2197 | void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); } | ||||||
2198 | |||||||
2199 | // Floating point division is side-effect free. We can not require that the | ||||||
2200 | // divisor is fully initialized and must propagate shadow. See PR37523. | ||||||
2201 | void visitFDiv(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2202 | void visitFRem(BinaryOperator &I) { handleShadowOr(I); } | ||||||
2203 | |||||||
2204 | /// Instrument == and != comparisons. | ||||||
2205 | /// | ||||||
2206 | /// Sometimes the comparison result is known even if some of the bits of the | ||||||
2207 | /// arguments are not. | ||||||
2208 | void handleEqualityComparison(ICmpInst &I) { | ||||||
2209 | IRBuilder<> IRB(&I); | ||||||
2210 | Value *A = I.getOperand(0); | ||||||
2211 | Value *B = I.getOperand(1); | ||||||
2212 | Value *Sa = getShadow(A); | ||||||
2213 | Value *Sb = getShadow(B); | ||||||
2214 | |||||||
2215 | // Get rid of pointers and vectors of pointers. | ||||||
2216 | // For ints (and vectors of ints), types of A and Sa match, | ||||||
2217 | // and this is a no-op. | ||||||
2218 | A = IRB.CreatePointerCast(A, Sa->getType()); | ||||||
2219 | B = IRB.CreatePointerCast(B, Sb->getType()); | ||||||
2220 | |||||||
2221 | // A == B <==> (C = A^B) == 0 | ||||||
2222 | // A != B <==> (C = A^B) != 0 | ||||||
2223 | // Sc = Sa | Sb | ||||||
2224 | Value *C = IRB.CreateXor(A, B); | ||||||
2225 | Value *Sc = IRB.CreateOr(Sa, Sb); | ||||||
2226 | // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now) | ||||||
2227 | // Result is defined if one of the following is true | ||||||
2228 | // * there is a defined 1 bit in C | ||||||
2229 | // * C is fully defined | ||||||
2230 | // Si = !(C & ~Sc) && Sc | ||||||
2231 | Value *Zero = Constant::getNullValue(Sc->getType()); | ||||||
2232 | Value *MinusOne = Constant::getAllOnesValue(Sc->getType()); | ||||||
2233 | Value *Si = | ||||||
2234 | IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero), | ||||||
2235 | IRB.CreateICmpEQ( | ||||||
2236 | IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero)); | ||||||
2237 | Si->setName("_msprop_icmp"); | ||||||
2238 | setShadow(&I, Si); | ||||||
2239 | setOriginForNaryOp(I); | ||||||
2240 | } | ||||||
2241 | |||||||
2242 | /// Build the lowest possible value of V, taking into account V's | ||||||
2243 | /// uninitialized bits. | ||||||
2244 | Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, | ||||||
2245 | bool isSigned) { | ||||||
2246 | if (isSigned) { | ||||||
2247 | // Split shadow into sign bit and other bits. | ||||||
2248 | Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); | ||||||
2249 | Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); | ||||||
2250 | // Maximise the undefined shadow bit, minimize other undefined bits. | ||||||
2251 | return | ||||||
2252 | IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit); | ||||||
2253 | } else { | ||||||
2254 | // Minimize undefined bits. | ||||||
2255 | return IRB.CreateAnd(A, IRB.CreateNot(Sa)); | ||||||
2256 | } | ||||||
2257 | } | ||||||
2258 | |||||||
2259 | /// Build the highest possible value of V, taking into account V's | ||||||
2260 | /// uninitialized bits. | ||||||
2261 | Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa, | ||||||
2262 | bool isSigned) { | ||||||
2263 | if (isSigned) { | ||||||
2264 | // Split shadow into sign bit and other bits. | ||||||
2265 | Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1); | ||||||
2266 | Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits); | ||||||
2267 | // Minimise the undefined shadow bit, maximise other undefined bits. | ||||||
2268 | return | ||||||
2269 | IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits); | ||||||
2270 | } else { | ||||||
2271 | // Maximize undefined bits. | ||||||
2272 | return IRB.CreateOr(A, Sa); | ||||||
2273 | } | ||||||
2274 | } | ||||||
2275 | |||||||
2276 | /// Instrument relational comparisons. | ||||||
2277 | /// | ||||||
2278 | /// This function does exact shadow propagation for all relational | ||||||
2279 | /// comparisons of integers, pointers and vectors of those. | ||||||
2280 | /// FIXME: output seems suboptimal when one of the operands is a constant | ||||||
2281 | void handleRelationalComparisonExact(ICmpInst &I) { | ||||||
2282 | IRBuilder<> IRB(&I); | ||||||
2283 | Value *A = I.getOperand(0); | ||||||
2284 | Value *B = I.getOperand(1); | ||||||
2285 | Value *Sa = getShadow(A); | ||||||
2286 | Value *Sb = getShadow(B); | ||||||
2287 | |||||||
2288 | // Get rid of pointers and vectors of pointers. | ||||||
2289 | // For ints (and vectors of ints), types of A and Sa match, | ||||||
2290 | // and this is a no-op. | ||||||
2291 | A = IRB.CreatePointerCast(A, Sa->getType()); | ||||||
2292 | B = IRB.CreatePointerCast(B, Sb->getType()); | ||||||
2293 | |||||||
2294 | // Let [a0, a1] be the interval of possible values of A, taking into account | ||||||
2295 | // its undefined bits. Let [b0, b1] be the interval of possible values of B. | ||||||
2296 | // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0). | ||||||
2297 | bool IsSigned = I.isSigned(); | ||||||
2298 | Value *S1 = IRB.CreateICmp(I.getPredicate(), | ||||||
2299 | getLowestPossibleValue(IRB, A, Sa, IsSigned), | ||||||
2300 | getHighestPossibleValue(IRB, B, Sb, IsSigned)); | ||||||
2301 | Value *S2 = IRB.CreateICmp(I.getPredicate(), | ||||||
2302 | getHighestPossibleValue(IRB, A, Sa, IsSigned), | ||||||
2303 | getLowestPossibleValue(IRB, B, Sb, IsSigned)); | ||||||
2304 | Value *Si = IRB.CreateXor(S1, S2); | ||||||
2305 | setShadow(&I, Si); | ||||||
2306 | setOriginForNaryOp(I); | ||||||
2307 | } | ||||||
2308 | |||||||
2309 | /// Instrument signed relational comparisons. | ||||||
2310 | /// | ||||||
2311 | /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest | ||||||
2312 | /// bit of the shadow. Everything else is delegated to handleShadowOr(). | ||||||
2313 | void handleSignedRelationalComparison(ICmpInst &I) { | ||||||
2314 | Constant *constOp; | ||||||
2315 | Value *op = nullptr; | ||||||
2316 | CmpInst::Predicate pre; | ||||||
2317 | if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) { | ||||||
2318 | op = I.getOperand(0); | ||||||
2319 | pre = I.getPredicate(); | ||||||
2320 | } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) { | ||||||
2321 | op = I.getOperand(1); | ||||||
2322 | pre = I.getSwappedPredicate(); | ||||||
2323 | } else { | ||||||
2324 | handleShadowOr(I); | ||||||
2325 | return; | ||||||
2326 | } | ||||||
2327 | |||||||
2328 | if ((constOp->isNullValue() && | ||||||
2329 | (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) || | ||||||
2330 | (constOp->isAllOnesValue() && | ||||||
2331 | (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) { | ||||||
2332 | IRBuilder<> IRB(&I); | ||||||
2333 | Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op), | ||||||
2334 | "_msprop_icmp_s"); | ||||||
2335 | setShadow(&I, Shadow); | ||||||
2336 | setOrigin(&I, getOrigin(op)); | ||||||
2337 | } else { | ||||||
2338 | handleShadowOr(I); | ||||||
2339 | } | ||||||
2340 | } | ||||||
2341 | |||||||
2342 | void visitICmpInst(ICmpInst &I) { | ||||||
2343 | if (!ClHandleICmp) { | ||||||
2344 | handleShadowOr(I); | ||||||
2345 | return; | ||||||
2346 | } | ||||||
2347 | if (I.isEquality()) { | ||||||
2348 | handleEqualityComparison(I); | ||||||
2349 | return; | ||||||
2350 | } | ||||||
2351 | |||||||
2352 | assert(I.isRelational())((I.isRelational()) ? static_cast<void> (0) : __assert_fail ("I.isRelational()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2352, __PRETTY_FUNCTION__)); | ||||||
2353 | if (ClHandleICmpExact) { | ||||||
2354 | handleRelationalComparisonExact(I); | ||||||
2355 | return; | ||||||
2356 | } | ||||||
2357 | if (I.isSigned()) { | ||||||
2358 | handleSignedRelationalComparison(I); | ||||||
2359 | return; | ||||||
2360 | } | ||||||
2361 | |||||||
2362 | assert(I.isUnsigned())((I.isUnsigned()) ? static_cast<void> (0) : __assert_fail ("I.isUnsigned()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2362, __PRETTY_FUNCTION__)); | ||||||
2363 | if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) { | ||||||
2364 | handleRelationalComparisonExact(I); | ||||||
2365 | return; | ||||||
2366 | } | ||||||
2367 | |||||||
2368 | handleShadowOr(I); | ||||||
2369 | } | ||||||
2370 | |||||||
2371 | void visitFCmpInst(FCmpInst &I) { | ||||||
2372 | handleShadowOr(I); | ||||||
2373 | } | ||||||
2374 | |||||||
2375 | void handleShift(BinaryOperator &I) { | ||||||
2376 | IRBuilder<> IRB(&I); | ||||||
2377 | // If any of the S2 bits are poisoned, the whole thing is poisoned. | ||||||
2378 | // Otherwise perform the same shift on S1. | ||||||
2379 | Value *S1 = getShadow(&I, 0); | ||||||
2380 | Value *S2 = getShadow(&I, 1); | ||||||
2381 | Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), | ||||||
2382 | S2->getType()); | ||||||
2383 | Value *V2 = I.getOperand(1); | ||||||
2384 | Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2); | ||||||
2385 | setShadow(&I, IRB.CreateOr(Shift, S2Conv)); | ||||||
2386 | setOriginForNaryOp(I); | ||||||
2387 | } | ||||||
2388 | |||||||
2389 | void visitShl(BinaryOperator &I) { handleShift(I); } | ||||||
2390 | void visitAShr(BinaryOperator &I) { handleShift(I); } | ||||||
2391 | void visitLShr(BinaryOperator &I) { handleShift(I); } | ||||||
2392 | |||||||
2393 | /// Instrument llvm.memmove | ||||||
2394 | /// | ||||||
2395 | /// At this point we don't know if llvm.memmove will be inlined or not. | ||||||
2396 | /// If we don't instrument it and it gets inlined, | ||||||
2397 | /// our interceptor will not kick in and we will lose the memmove. | ||||||
2398 | /// If we instrument the call here, but it does not get inlined, | ||||||
2399 | /// we will memove the shadow twice: which is bad in case | ||||||
2400 | /// of overlapping regions. So, we simply lower the intrinsic to a call. | ||||||
2401 | /// | ||||||
2402 | /// Similar situation exists for memcpy and memset. | ||||||
2403 | void visitMemMoveInst(MemMoveInst &I) { | ||||||
2404 | IRBuilder<> IRB(&I); | ||||||
2405 | IRB.CreateCall( | ||||||
2406 | MS.MemmoveFn, | ||||||
2407 | {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()), | ||||||
2408 | IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()), | ||||||
2409 | IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)}); | ||||||
2410 | I.eraseFromParent(); | ||||||
2411 | } | ||||||
2412 | |||||||
2413 | // Similar to memmove: avoid copying shadow twice. | ||||||
2414 | // This is somewhat unfortunate as it may slowdown small constant memcpys. | ||||||
2415 | // FIXME: consider doing manual inline for small constant sizes and proper | ||||||
2416 | // alignment. | ||||||
2417 | void visitMemCpyInst(MemCpyInst &I) { | ||||||
2418 | IRBuilder<> IRB(&I); | ||||||
2419 | IRB.CreateCall( | ||||||
2420 | MS.MemcpyFn, | ||||||
2421 | {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()), | ||||||
2422 | IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()), | ||||||
2423 | IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)}); | ||||||
2424 | I.eraseFromParent(); | ||||||
2425 | } | ||||||
2426 | |||||||
2427 | // Same as memcpy. | ||||||
2428 | void visitMemSetInst(MemSetInst &I) { | ||||||
2429 | IRBuilder<> IRB(&I); | ||||||
2430 | IRB.CreateCall( | ||||||
2431 | MS.MemsetFn, | ||||||
2432 | {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()), | ||||||
2433 | IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false), | ||||||
2434 | IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)}); | ||||||
2435 | I.eraseFromParent(); | ||||||
2436 | } | ||||||
2437 | |||||||
2438 | void visitVAStartInst(VAStartInst &I) { | ||||||
2439 | VAHelper->visitVAStartInst(I); | ||||||
2440 | } | ||||||
2441 | |||||||
2442 | void visitVACopyInst(VACopyInst &I) { | ||||||
2443 | VAHelper->visitVACopyInst(I); | ||||||
2444 | } | ||||||
2445 | |||||||
2446 | /// Handle vector store-like intrinsics. | ||||||
2447 | /// | ||||||
2448 | /// Instrument intrinsics that look like a simple SIMD store: writes memory, | ||||||
2449 | /// has 1 pointer argument and 1 vector argument, returns void. | ||||||
2450 | bool handleVectorStoreIntrinsic(IntrinsicInst &I) { | ||||||
2451 | IRBuilder<> IRB(&I); | ||||||
2452 | Value* Addr = I.getArgOperand(0); | ||||||
2453 | Value *Shadow = getShadow(&I, 1); | ||||||
2454 | Value *ShadowPtr, *OriginPtr; | ||||||
2455 | |||||||
2456 | // We don't know the pointer alignment (could be unaligned SSE store!). | ||||||
2457 | // Have to assume to worst case. | ||||||
2458 | std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr( | ||||||
2459 | Addr, IRB, Shadow->getType(), Align::None(), /*isStore*/ true); | ||||||
2460 | IRB.CreateAlignedStore(Shadow, ShadowPtr, 1); | ||||||
2461 | |||||||
2462 | if (ClCheckAccessAddress) | ||||||
2463 | insertShadowCheck(Addr, &I); | ||||||
2464 | |||||||
2465 | // FIXME: factor out common code from materializeStores | ||||||
2466 | if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr); | ||||||
2467 | return true; | ||||||
2468 | } | ||||||
2469 | |||||||
2470 | /// Handle vector load-like intrinsics. | ||||||
2471 | /// | ||||||
2472 | /// Instrument intrinsics that look like a simple SIMD load: reads memory, | ||||||
2473 | /// has 1 pointer argument, returns a vector. | ||||||
2474 | bool handleVectorLoadIntrinsic(IntrinsicInst &I) { | ||||||
2475 | IRBuilder<> IRB(&I); | ||||||
2476 | Value *Addr = I.getArgOperand(0); | ||||||
2477 | |||||||
2478 | Type *ShadowTy = getShadowTy(&I); | ||||||
2479 | Value *ShadowPtr = nullptr, *OriginPtr = nullptr; | ||||||
2480 | if (PropagateShadow) { | ||||||
2481 | // We don't know the pointer alignment (could be unaligned SSE load!). | ||||||
2482 | // Have to assume to worst case. | ||||||
2483 | const Align Alignment = Align::None(); | ||||||
2484 | std::tie(ShadowPtr, OriginPtr) = | ||||||
2485 | getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); | ||||||
2486 | setShadow(&I, IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, | ||||||
2487 | Alignment.value(), "_msld")); | ||||||
2488 | } else { | ||||||
2489 | setShadow(&I, getCleanShadow(&I)); | ||||||
2490 | } | ||||||
2491 | |||||||
2492 | if (ClCheckAccessAddress) | ||||||
2493 | insertShadowCheck(Addr, &I); | ||||||
2494 | |||||||
2495 | if (MS.TrackOrigins) { | ||||||
2496 | if (PropagateShadow) | ||||||
2497 | setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr)); | ||||||
2498 | else | ||||||
2499 | setOrigin(&I, getCleanOrigin()); | ||||||
2500 | } | ||||||
2501 | return true; | ||||||
2502 | } | ||||||
2503 | |||||||
2504 | /// Handle (SIMD arithmetic)-like intrinsics. | ||||||
2505 | /// | ||||||
2506 | /// Instrument intrinsics with any number of arguments of the same type, | ||||||
2507 | /// equal to the return type. The type should be simple (no aggregates or | ||||||
2508 | /// pointers; vectors are fine). | ||||||
2509 | /// Caller guarantees that this intrinsic does not access memory. | ||||||
2510 | bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) { | ||||||
2511 | Type *RetTy = I.getType(); | ||||||
2512 | if (!(RetTy->isIntOrIntVectorTy() || | ||||||
2513 | RetTy->isFPOrFPVectorTy() || | ||||||
2514 | RetTy->isX86_MMXTy())) | ||||||
2515 | return false; | ||||||
2516 | |||||||
2517 | unsigned NumArgOperands = I.getNumArgOperands(); | ||||||
2518 | |||||||
2519 | for (unsigned i = 0; i < NumArgOperands; ++i) { | ||||||
2520 | Type *Ty = I.getArgOperand(i)->getType(); | ||||||
2521 | if (Ty != RetTy) | ||||||
2522 | return false; | ||||||
2523 | } | ||||||
2524 | |||||||
2525 | IRBuilder<> IRB(&I); | ||||||
2526 | ShadowAndOriginCombiner SC(this, IRB); | ||||||
2527 | for (unsigned i = 0; i < NumArgOperands; ++i) | ||||||
2528 | SC.Add(I.getArgOperand(i)); | ||||||
2529 | SC.Done(&I); | ||||||
2530 | |||||||
2531 | return true; | ||||||
2532 | } | ||||||
2533 | |||||||
2534 | /// Heuristically instrument unknown intrinsics. | ||||||
2535 | /// | ||||||
2536 | /// The main purpose of this code is to do something reasonable with all | ||||||
2537 | /// random intrinsics we might encounter, most importantly - SIMD intrinsics. | ||||||
2538 | /// We recognize several classes of intrinsics by their argument types and | ||||||
2539 | /// ModRefBehaviour and apply special intrumentation when we are reasonably | ||||||
2540 | /// sure that we know what the intrinsic does. | ||||||
2541 | /// | ||||||
2542 | /// We special-case intrinsics where this approach fails. See llvm.bswap | ||||||
2543 | /// handling as an example of that. | ||||||
2544 | bool handleUnknownIntrinsic(IntrinsicInst &I) { | ||||||
2545 | unsigned NumArgOperands = I.getNumArgOperands(); | ||||||
2546 | if (NumArgOperands == 0) | ||||||
2547 | return false; | ||||||
2548 | |||||||
2549 | if (NumArgOperands == 2 && | ||||||
2550 | I.getArgOperand(0)->getType()->isPointerTy() && | ||||||
2551 | I.getArgOperand(1)->getType()->isVectorTy() && | ||||||
2552 | I.getType()->isVoidTy() && | ||||||
2553 | !I.onlyReadsMemory()) { | ||||||
2554 | // This looks like a vector store. | ||||||
2555 | return handleVectorStoreIntrinsic(I); | ||||||
2556 | } | ||||||
2557 | |||||||
2558 | if (NumArgOperands == 1 && | ||||||
2559 | I.getArgOperand(0)->getType()->isPointerTy() && | ||||||
2560 | I.getType()->isVectorTy() && | ||||||
2561 | I.onlyReadsMemory()) { | ||||||
2562 | // This looks like a vector load. | ||||||
2563 | return handleVectorLoadIntrinsic(I); | ||||||
2564 | } | ||||||
2565 | |||||||
2566 | if (I.doesNotAccessMemory()) | ||||||
2567 | if (maybeHandleSimpleNomemIntrinsic(I)) | ||||||
2568 | return true; | ||||||
2569 | |||||||
2570 | // FIXME: detect and handle SSE maskstore/maskload | ||||||
2571 | return false; | ||||||
2572 | } | ||||||
2573 | |||||||
2574 | void handleInvariantGroup(IntrinsicInst &I) { | ||||||
2575 | setShadow(&I, getShadow(&I, 0)); | ||||||
2576 | setOrigin(&I, getOrigin(&I, 0)); | ||||||
2577 | } | ||||||
2578 | |||||||
2579 | void handleLifetimeStart(IntrinsicInst &I) { | ||||||
2580 | if (!PoisonStack) | ||||||
2581 | return; | ||||||
2582 | DenseMap<Value *, AllocaInst *> AllocaForValue; | ||||||
2583 | AllocaInst *AI = | ||||||
2584 | llvm::findAllocaForValue(I.getArgOperand(1), AllocaForValue); | ||||||
2585 | if (!AI) | ||||||
2586 | InstrumentLifetimeStart = false; | ||||||
2587 | LifetimeStartList.push_back(std::make_pair(&I, AI)); | ||||||
2588 | } | ||||||
2589 | |||||||
2590 | void handleBswap(IntrinsicInst &I) { | ||||||
2591 | IRBuilder<> IRB(&I); | ||||||
2592 | Value *Op = I.getArgOperand(0); | ||||||
2593 | Type *OpType = Op->getType(); | ||||||
2594 | Function *BswapFunc = Intrinsic::getDeclaration( | ||||||
2595 | F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1)); | ||||||
2596 | setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op))); | ||||||
2597 | setOrigin(&I, getOrigin(Op)); | ||||||
2598 | } | ||||||
2599 | |||||||
2600 | // Instrument vector convert instrinsic. | ||||||
2601 | // | ||||||
2602 | // This function instruments intrinsics like cvtsi2ss: | ||||||
2603 | // %Out = int_xxx_cvtyyy(%ConvertOp) | ||||||
2604 | // or | ||||||
2605 | // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp) | ||||||
2606 | // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same | ||||||
2607 | // number \p Out elements, and (if has 2 arguments) copies the rest of the | ||||||
2608 | // elements from \p CopyOp. | ||||||
2609 | // In most cases conversion involves floating-point value which may trigger a | ||||||
2610 | // hardware exception when not fully initialized. For this reason we require | ||||||
2611 | // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise. | ||||||
2612 | // We copy the shadow of \p CopyOp[NumUsedElements:] to \p | ||||||
2613 | // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always | ||||||
2614 | // return a fully initialized value. | ||||||
2615 | void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) { | ||||||
2616 | IRBuilder<> IRB(&I); | ||||||
2617 | Value *CopyOp, *ConvertOp; | ||||||
2618 | |||||||
2619 | switch (I.getNumArgOperands()) { | ||||||
2620 | case 3: | ||||||
2621 | assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode")((isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode" ) ? static_cast<void> (0) : __assert_fail ("isa<ConstantInt>(I.getArgOperand(2)) && \"Invalid rounding mode\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2621, __PRETTY_FUNCTION__)); | ||||||
2622 | LLVM_FALLTHROUGH[[gnu::fallthrough]]; | ||||||
2623 | case 2: | ||||||
2624 | CopyOp = I.getArgOperand(0); | ||||||
2625 | ConvertOp = I.getArgOperand(1); | ||||||
2626 | break; | ||||||
2627 | case 1: | ||||||
2628 | ConvertOp = I.getArgOperand(0); | ||||||
2629 | CopyOp = nullptr; | ||||||
2630 | break; | ||||||
2631 | default: | ||||||
2632 | llvm_unreachable("Cvt intrinsic with unsupported number of arguments.")::llvm::llvm_unreachable_internal("Cvt intrinsic with unsupported number of arguments." , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2632); | ||||||
2633 | } | ||||||
2634 | |||||||
2635 | // The first *NumUsedElements* elements of ConvertOp are converted to the | ||||||
2636 | // same number of output elements. The rest of the output is copied from | ||||||
2637 | // CopyOp, or (if not available) filled with zeroes. | ||||||
2638 | // Combine shadow for elements of ConvertOp that are used in this operation, | ||||||
2639 | // and insert a check. | ||||||
2640 | // FIXME: consider propagating shadow of ConvertOp, at least in the case of | ||||||
2641 | // int->any conversion. | ||||||
2642 | Value *ConvertShadow = getShadow(ConvertOp); | ||||||
2643 | Value *AggShadow = nullptr; | ||||||
2644 | if (ConvertOp->getType()->isVectorTy()) { | ||||||
2645 | AggShadow = IRB.CreateExtractElement( | ||||||
2646 | ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0)); | ||||||
2647 | for (int i = 1; i < NumUsedElements; ++i) { | ||||||
2648 | Value *MoreShadow = IRB.CreateExtractElement( | ||||||
2649 | ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i)); | ||||||
2650 | AggShadow = IRB.CreateOr(AggShadow, MoreShadow); | ||||||
2651 | } | ||||||
2652 | } else { | ||||||
2653 | AggShadow = ConvertShadow; | ||||||
2654 | } | ||||||
2655 | assert(AggShadow->getType()->isIntegerTy())((AggShadow->getType()->isIntegerTy()) ? static_cast< void> (0) : __assert_fail ("AggShadow->getType()->isIntegerTy()" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2655, __PRETTY_FUNCTION__)); | ||||||
2656 | insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I); | ||||||
2657 | |||||||
2658 | // Build result shadow by zero-filling parts of CopyOp shadow that come from | ||||||
2659 | // ConvertOp. | ||||||
2660 | if (CopyOp) { | ||||||
2661 | assert(CopyOp->getType() == I.getType())((CopyOp->getType() == I.getType()) ? static_cast<void> (0) : __assert_fail ("CopyOp->getType() == I.getType()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2661, __PRETTY_FUNCTION__)); | ||||||
2662 | assert(CopyOp->getType()->isVectorTy())((CopyOp->getType()->isVectorTy()) ? static_cast<void > (0) : __assert_fail ("CopyOp->getType()->isVectorTy()" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2662, __PRETTY_FUNCTION__)); | ||||||
2663 | Value *ResultShadow = getShadow(CopyOp); | ||||||
2664 | Type *EltTy = ResultShadow->getType()->getVectorElementType(); | ||||||
2665 | for (int i = 0; i < NumUsedElements; ++i) { | ||||||
2666 | ResultShadow = IRB.CreateInsertElement( | ||||||
2667 | ResultShadow, ConstantInt::getNullValue(EltTy), | ||||||
2668 | ConstantInt::get(IRB.getInt32Ty(), i)); | ||||||
2669 | } | ||||||
2670 | setShadow(&I, ResultShadow); | ||||||
2671 | setOrigin(&I, getOrigin(CopyOp)); | ||||||
2672 | } else { | ||||||
2673 | setShadow(&I, getCleanShadow(&I)); | ||||||
2674 | setOrigin(&I, getCleanOrigin()); | ||||||
2675 | } | ||||||
2676 | } | ||||||
2677 | |||||||
2678 | // Given a scalar or vector, extract lower 64 bits (or less), and return all | ||||||
2679 | // zeroes if it is zero, and all ones otherwise. | ||||||
2680 | Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) { | ||||||
2681 | if (S->getType()->isVectorTy()) | ||||||
2682 | S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true); | ||||||
2683 | assert(S->getType()->getPrimitiveSizeInBits() <= 64)((S->getType()->getPrimitiveSizeInBits() <= 64) ? static_cast <void> (0) : __assert_fail ("S->getType()->getPrimitiveSizeInBits() <= 64" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2683, __PRETTY_FUNCTION__)); | ||||||
2684 | Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S)); | ||||||
2685 | return CreateShadowCast(IRB, S2, T, /* Signed */ true); | ||||||
2686 | } | ||||||
2687 | |||||||
2688 | // Given a vector, extract its first element, and return all | ||||||
2689 | // zeroes if it is zero, and all ones otherwise. | ||||||
2690 | Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) { | ||||||
2691 | Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0); | ||||||
2692 | Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1)); | ||||||
2693 | return CreateShadowCast(IRB, S2, T, /* Signed */ true); | ||||||
2694 | } | ||||||
2695 | |||||||
2696 | Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) { | ||||||
2697 | Type *T = S->getType(); | ||||||
2698 | assert(T->isVectorTy())((T->isVectorTy()) ? static_cast<void> (0) : __assert_fail ("T->isVectorTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2698, __PRETTY_FUNCTION__)); | ||||||
2699 | Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S)); | ||||||
2700 | return IRB.CreateSExt(S2, T); | ||||||
2701 | } | ||||||
2702 | |||||||
2703 | // Instrument vector shift instrinsic. | ||||||
2704 | // | ||||||
2705 | // This function instruments intrinsics like int_x86_avx2_psll_w. | ||||||
2706 | // Intrinsic shifts %In by %ShiftSize bits. | ||||||
2707 | // %ShiftSize may be a vector. In that case the lower 64 bits determine shift | ||||||
2708 | // size, and the rest is ignored. Behavior is defined even if shift size is | ||||||
2709 | // greater than register (or field) width. | ||||||
2710 | void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) { | ||||||
2711 | assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) : __assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2711, __PRETTY_FUNCTION__)); | ||||||
2712 | IRBuilder<> IRB(&I); | ||||||
2713 | // If any of the S2 bits are poisoned, the whole thing is poisoned. | ||||||
2714 | // Otherwise perform the same shift on S1. | ||||||
2715 | Value *S1 = getShadow(&I, 0); | ||||||
2716 | Value *S2 = getShadow(&I, 1); | ||||||
2717 | Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2) | ||||||
2718 | : Lower64ShadowExtend(IRB, S2, getShadowTy(&I)); | ||||||
2719 | Value *V1 = I.getOperand(0); | ||||||
2720 | Value *V2 = I.getOperand(1); | ||||||
2721 | Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledValue(), | ||||||
2722 | {IRB.CreateBitCast(S1, V1->getType()), V2}); | ||||||
2723 | Shift = IRB.CreateBitCast(Shift, getShadowTy(&I)); | ||||||
2724 | setShadow(&I, IRB.CreateOr(Shift, S2Conv)); | ||||||
2725 | setOriginForNaryOp(I); | ||||||
2726 | } | ||||||
2727 | |||||||
2728 | // Get an X86_MMX-sized vector type. | ||||||
2729 | Type *getMMXVectorTy(unsigned EltSizeInBits) { | ||||||
2730 | const unsigned X86_MMXSizeInBits = 64; | ||||||
2731 | assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits ) == 0 && "Illegal MMX vector element size") ? static_cast <void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2732, __PRETTY_FUNCTION__)) | ||||||
2732 | "Illegal MMX vector element size")((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits ) == 0 && "Illegal MMX vector element size") ? static_cast <void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2732, __PRETTY_FUNCTION__)); | ||||||
2733 | return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits), | ||||||
2734 | X86_MMXSizeInBits / EltSizeInBits); | ||||||
2735 | } | ||||||
2736 | |||||||
2737 | // Returns a signed counterpart for an (un)signed-saturate-and-pack | ||||||
2738 | // intrinsic. | ||||||
2739 | Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) { | ||||||
2740 | switch (id) { | ||||||
2741 | case Intrinsic::x86_sse2_packsswb_128: | ||||||
2742 | case Intrinsic::x86_sse2_packuswb_128: | ||||||
2743 | return Intrinsic::x86_sse2_packsswb_128; | ||||||
2744 | |||||||
2745 | case Intrinsic::x86_sse2_packssdw_128: | ||||||
2746 | case Intrinsic::x86_sse41_packusdw: | ||||||
2747 | return Intrinsic::x86_sse2_packssdw_128; | ||||||
2748 | |||||||
2749 | case Intrinsic::x86_avx2_packsswb: | ||||||
2750 | case Intrinsic::x86_avx2_packuswb: | ||||||
2751 | return Intrinsic::x86_avx2_packsswb; | ||||||
2752 | |||||||
2753 | case Intrinsic::x86_avx2_packssdw: | ||||||
2754 | case Intrinsic::x86_avx2_packusdw: | ||||||
2755 | return Intrinsic::x86_avx2_packssdw; | ||||||
2756 | |||||||
2757 | case Intrinsic::x86_mmx_packsswb: | ||||||
2758 | case Intrinsic::x86_mmx_packuswb: | ||||||
2759 | return Intrinsic::x86_mmx_packsswb; | ||||||
2760 | |||||||
2761 | case Intrinsic::x86_mmx_packssdw: | ||||||
2762 | return Intrinsic::x86_mmx_packssdw; | ||||||
2763 | default: | ||||||
2764 | llvm_unreachable("unexpected intrinsic id")::llvm::llvm_unreachable_internal("unexpected intrinsic id", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2764); | ||||||
2765 | } | ||||||
2766 | } | ||||||
2767 | |||||||
2768 | // Instrument vector pack instrinsic. | ||||||
2769 | // | ||||||
2770 | // This function instruments intrinsics like x86_mmx_packsswb, that | ||||||
2771 | // packs elements of 2 input vectors into half as many bits with saturation. | ||||||
2772 | // Shadow is propagated with the signed variant of the same intrinsic applied | ||||||
2773 | // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer). | ||||||
2774 | // EltSizeInBits is used only for x86mmx arguments. | ||||||
2775 | void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) { | ||||||
2776 | assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) : __assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2776, __PRETTY_FUNCTION__)); | ||||||
2777 | bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy(); | ||||||
2778 | IRBuilder<> IRB(&I); | ||||||
2779 | Value *S1 = getShadow(&I, 0); | ||||||
2780 | Value *S2 = getShadow(&I, 1); | ||||||
2781 | assert(isX86_MMX || S1->getType()->isVectorTy())((isX86_MMX || S1->getType()->isVectorTy()) ? static_cast <void> (0) : __assert_fail ("isX86_MMX || S1->getType()->isVectorTy()" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 2781, __PRETTY_FUNCTION__)); | ||||||
2782 | |||||||
2783 | // SExt and ICmpNE below must apply to individual elements of input vectors. | ||||||
2784 | // In case of x86mmx arguments, cast them to appropriate vector types and | ||||||
2785 | // back. | ||||||
2786 | Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType(); | ||||||
2787 | if (isX86_MMX) { | ||||||
2788 | S1 = IRB.CreateBitCast(S1, T); | ||||||
2789 | S2 = IRB.CreateBitCast(S2, T); | ||||||
2790 | } | ||||||
2791 | Value *S1_ext = IRB.CreateSExt( | ||||||
2792 | IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T); | ||||||
2793 | Value *S2_ext = IRB.CreateSExt( | ||||||
2794 | IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T); | ||||||
2795 | if (isX86_MMX) { | ||||||
2796 | Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C); | ||||||
2797 | S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy); | ||||||
2798 | S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy); | ||||||
2799 | } | ||||||
2800 | |||||||
2801 | Function *ShadowFn = Intrinsic::getDeclaration( | ||||||
2802 | F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID())); | ||||||
2803 | |||||||
2804 | Value *S = | ||||||
2805 | IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack"); | ||||||
2806 | if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I)); | ||||||
2807 | setShadow(&I, S); | ||||||
2808 | setOriginForNaryOp(I); | ||||||
2809 | } | ||||||
2810 | |||||||
2811 | // Instrument sum-of-absolute-differencies intrinsic. | ||||||
2812 | void handleVectorSadIntrinsic(IntrinsicInst &I) { | ||||||
2813 | const unsigned SignificantBitsPerResultElement = 16; | ||||||
2814 | bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy(); | ||||||
2815 | Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType(); | ||||||
2816 | unsigned ZeroBitsPerResultElement = | ||||||
2817 | ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement; | ||||||
2818 | |||||||
2819 | IRBuilder<> IRB(&I); | ||||||
2820 | Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1)); | ||||||
2821 | S = IRB.CreateBitCast(S, ResTy); | ||||||
2822 | S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)), | ||||||
2823 | ResTy); | ||||||
2824 | S = IRB.CreateLShr(S, ZeroBitsPerResultElement); | ||||||
2825 | S = IRB.CreateBitCast(S, getShadowTy(&I)); | ||||||
2826 | setShadow(&I, S); | ||||||
2827 | setOriginForNaryOp(I); | ||||||
2828 | } | ||||||
2829 | |||||||
2830 | // Instrument multiply-add intrinsic. | ||||||
2831 | void handleVectorPmaddIntrinsic(IntrinsicInst &I, | ||||||
2832 | unsigned EltSizeInBits = 0) { | ||||||
2833 | bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy(); | ||||||
2834 | Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType(); | ||||||
2835 | IRBuilder<> IRB(&I); | ||||||
2836 | Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1)); | ||||||
2837 | S = IRB.CreateBitCast(S, ResTy); | ||||||
2838 | S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)), | ||||||
2839 | ResTy); | ||||||
2840 | S = IRB.CreateBitCast(S, getShadowTy(&I)); | ||||||
2841 | setShadow(&I, S); | ||||||
2842 | setOriginForNaryOp(I); | ||||||
2843 | } | ||||||
2844 | |||||||
2845 | // Instrument compare-packed intrinsic. | ||||||
2846 | // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or | ||||||
2847 | // all-ones shadow. | ||||||
2848 | void handleVectorComparePackedIntrinsic(IntrinsicInst &I) { | ||||||
2849 | IRBuilder<> IRB(&I); | ||||||
2850 | Type *ResTy = getShadowTy(&I); | ||||||
2851 | Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1)); | ||||||
2852 | Value *S = IRB.CreateSExt( | ||||||
2853 | IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy); | ||||||
2854 | setShadow(&I, S); | ||||||
2855 | setOriginForNaryOp(I); | ||||||
2856 | } | ||||||
2857 | |||||||
2858 | // Instrument compare-scalar intrinsic. | ||||||
2859 | // This handles both cmp* intrinsics which return the result in the first | ||||||
2860 | // element of a vector, and comi* which return the result as i32. | ||||||
2861 | void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) { | ||||||
2862 | IRBuilder<> IRB(&I); | ||||||
2863 | Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1)); | ||||||
2864 | Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I)); | ||||||
2865 | setShadow(&I, S); | ||||||
2866 | setOriginForNaryOp(I); | ||||||
2867 | } | ||||||
2868 | |||||||
2869 | void handleStmxcsr(IntrinsicInst &I) { | ||||||
2870 | IRBuilder<> IRB(&I); | ||||||
2871 | Value* Addr = I.getArgOperand(0); | ||||||
2872 | Type *Ty = IRB.getInt32Ty(); | ||||||
2873 | Value *ShadowPtr = | ||||||
2874 | getShadowOriginPtr(Addr, IRB, Ty, Align::None(), /*isStore*/ true) | ||||||
2875 | .first; | ||||||
2876 | |||||||
2877 | IRB.CreateStore(getCleanShadow(Ty), | ||||||
2878 | IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo())); | ||||||
2879 | |||||||
2880 | if (ClCheckAccessAddress) | ||||||
2881 | insertShadowCheck(Addr, &I); | ||||||
2882 | } | ||||||
2883 | |||||||
2884 | void handleLdmxcsr(IntrinsicInst &I) { | ||||||
2885 | if (!InsertChecks) return; | ||||||
2886 | |||||||
2887 | IRBuilder<> IRB(&I); | ||||||
2888 | Value *Addr = I.getArgOperand(0); | ||||||
2889 | Type *Ty = IRB.getInt32Ty(); | ||||||
2890 | const Align Alignment = Align::None(); | ||||||
2891 | Value *ShadowPtr, *OriginPtr; | ||||||
2892 | std::tie(ShadowPtr, OriginPtr) = | ||||||
2893 | getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false); | ||||||
2894 | |||||||
2895 | if (ClCheckAccessAddress) | ||||||
2896 | insertShadowCheck(Addr, &I); | ||||||
2897 | |||||||
2898 | Value *Shadow = | ||||||
2899 | IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment.value(), "_ldmxcsr"); | ||||||
2900 | Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr) | ||||||
2901 | : getCleanOrigin(); | ||||||
2902 | insertShadowCheck(Shadow, Origin, &I); | ||||||
2903 | } | ||||||
2904 | |||||||
2905 | void handleMaskedStore(IntrinsicInst &I) { | ||||||
2906 | IRBuilder<> IRB(&I); | ||||||
2907 | Value *V = I.getArgOperand(0); | ||||||
2908 | Value *Addr = I.getArgOperand(1); | ||||||
2909 | const MaybeAlign Alignment( | ||||||
2910 | cast<ConstantInt>(I.getArgOperand(2))->getZExtValue()); | ||||||
2911 | Value *Mask = I.getArgOperand(3); | ||||||
2912 | Value *Shadow = getShadow(V); | ||||||
2913 | |||||||
2914 | Value *ShadowPtr; | ||||||
2915 | Value *OriginPtr; | ||||||
2916 | std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr( | ||||||
2917 | Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true); | ||||||
2918 | |||||||
2919 | if (ClCheckAccessAddress) { | ||||||
2920 | insertShadowCheck(Addr, &I); | ||||||
2921 | // Uninitialized mask is kind of like uninitialized address, but not as | ||||||
2922 | // scary. | ||||||
2923 | insertShadowCheck(Mask, &I); | ||||||
2924 | } | ||||||
2925 | |||||||
2926 | IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment ? Alignment->value() : 0, | ||||||
2927 | Mask); | ||||||
2928 | |||||||
2929 | if (MS.TrackOrigins) { | ||||||
2930 | auto &DL = F.getParent()->getDataLayout(); | ||||||
2931 | paintOrigin(IRB, getOrigin(V), OriginPtr, | ||||||
2932 | DL.getTypeStoreSize(Shadow->getType()), | ||||||
2933 | llvm::max(Alignment, kMinOriginAlignment)); | ||||||
2934 | } | ||||||
2935 | } | ||||||
2936 | |||||||
2937 | bool handleMaskedLoad(IntrinsicInst &I) { | ||||||
2938 | IRBuilder<> IRB(&I); | ||||||
2939 | Value *Addr = I.getArgOperand(0); | ||||||
2940 | const MaybeAlign Alignment( | ||||||
2941 | cast<ConstantInt>(I.getArgOperand(1))->getZExtValue()); | ||||||
2942 | Value *Mask = I.getArgOperand(2); | ||||||
2943 | Value *PassThru = I.getArgOperand(3); | ||||||
2944 | |||||||
2945 | Type *ShadowTy = getShadowTy(&I); | ||||||
2946 | Value *ShadowPtr, *OriginPtr; | ||||||
2947 | if (PropagateShadow) { | ||||||
2948 | std::tie(ShadowPtr, OriginPtr) = | ||||||
2949 | getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false); | ||||||
2950 | setShadow(&I, IRB.CreateMaskedLoad( | ||||||
2951 | ShadowPtr, Alignment ? Alignment->value() : 0, Mask, | ||||||
2952 | getShadow(PassThru), "_msmaskedld")); | ||||||
2953 | } else { | ||||||
2954 | setShadow(&I, getCleanShadow(&I)); | ||||||
2955 | } | ||||||
2956 | |||||||
2957 | if (ClCheckAccessAddress) { | ||||||
2958 | insertShadowCheck(Addr, &I); | ||||||
2959 | insertShadowCheck(Mask, &I); | ||||||
2960 | } | ||||||
2961 | |||||||
2962 | if (MS.TrackOrigins) { | ||||||
2963 | if (PropagateShadow) { | ||||||
2964 | // Choose between PassThru's and the loaded value's origins. | ||||||
2965 | Value *MaskedPassThruShadow = IRB.CreateAnd( | ||||||
2966 | getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy)); | ||||||
2967 | |||||||
2968 | Value *Acc = IRB.CreateExtractElement( | ||||||
2969 | MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0)); | ||||||
2970 | for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N; | ||||||
2971 | ++i) { | ||||||
2972 | Value *More = IRB.CreateExtractElement( | ||||||
2973 | MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i)); | ||||||
2974 | Acc = IRB.CreateOr(Acc, More); | ||||||
2975 | } | ||||||
2976 | |||||||
2977 | Value *Origin = IRB.CreateSelect( | ||||||
2978 | IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())), | ||||||
2979 | getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr)); | ||||||
2980 | |||||||
2981 | setOrigin(&I, Origin); | ||||||
2982 | } else { | ||||||
2983 | setOrigin(&I, getCleanOrigin()); | ||||||
2984 | } | ||||||
2985 | } | ||||||
2986 | return true; | ||||||
2987 | } | ||||||
2988 | |||||||
2989 | // Instrument BMI / BMI2 intrinsics. | ||||||
2990 | // All of these intrinsics are Z = I(X, Y) | ||||||
2991 | // where the types of all operands and the result match, and are either i32 or i64. | ||||||
2992 | // The following instrumentation happens to work for all of them: | ||||||
2993 | // Sz = I(Sx, Y) | (sext (Sy != 0)) | ||||||
2994 | void handleBmiIntrinsic(IntrinsicInst &I) { | ||||||
2995 | IRBuilder<> IRB(&I); | ||||||
2996 | Type *ShadowTy = getShadowTy(&I); | ||||||
2997 | |||||||
2998 | // If any bit of the mask operand is poisoned, then the whole thing is. | ||||||
2999 | Value *SMask = getShadow(&I, 1); | ||||||
3000 | SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)), | ||||||
3001 | ShadowTy); | ||||||
3002 | // Apply the same intrinsic to the shadow of the first operand. | ||||||
3003 | Value *S = IRB.CreateCall(I.getCalledFunction(), | ||||||
3004 | {getShadow(&I, 0), I.getOperand(1)}); | ||||||
3005 | S = IRB.CreateOr(SMask, S); | ||||||
3006 | setShadow(&I, S); | ||||||
3007 | setOriginForNaryOp(I); | ||||||
3008 | } | ||||||
3009 | |||||||
3010 | void visitIntrinsicInst(IntrinsicInst &I) { | ||||||
3011 | switch (I.getIntrinsicID()) { | ||||||
3012 | case Intrinsic::lifetime_start: | ||||||
3013 | handleLifetimeStart(I); | ||||||
3014 | break; | ||||||
3015 | case Intrinsic::launder_invariant_group: | ||||||
3016 | case Intrinsic::strip_invariant_group: | ||||||
3017 | handleInvariantGroup(I); | ||||||
3018 | break; | ||||||
3019 | case Intrinsic::bswap: | ||||||
3020 | handleBswap(I); | ||||||
3021 | break; | ||||||
3022 | case Intrinsic::masked_store: | ||||||
3023 | handleMaskedStore(I); | ||||||
3024 | break; | ||||||
3025 | case Intrinsic::masked_load: | ||||||
3026 | handleMaskedLoad(I); | ||||||
3027 | break; | ||||||
3028 | case Intrinsic::x86_sse_stmxcsr: | ||||||
3029 | handleStmxcsr(I); | ||||||
3030 | break; | ||||||
3031 | case Intrinsic::x86_sse_ldmxcsr: | ||||||
3032 | handleLdmxcsr(I); | ||||||
3033 | break; | ||||||
3034 | case Intrinsic::x86_avx512_vcvtsd2usi64: | ||||||
3035 | case Intrinsic::x86_avx512_vcvtsd2usi32: | ||||||
3036 | case Intrinsic::x86_avx512_vcvtss2usi64: | ||||||
3037 | case Intrinsic::x86_avx512_vcvtss2usi32: | ||||||
3038 | case Intrinsic::x86_avx512_cvttss2usi64: | ||||||
3039 | case Intrinsic::x86_avx512_cvttss2usi: | ||||||
3040 | case Intrinsic::x86_avx512_cvttsd2usi64: | ||||||
3041 | case Intrinsic::x86_avx512_cvttsd2usi: | ||||||
3042 | case Intrinsic::x86_avx512_cvtusi2ss: | ||||||
3043 | case Intrinsic::x86_avx512_cvtusi642sd: | ||||||
3044 | case Intrinsic::x86_avx512_cvtusi642ss: | ||||||
3045 | case Intrinsic::x86_sse2_cvtsd2si64: | ||||||
3046 | case Intrinsic::x86_sse2_cvtsd2si: | ||||||
3047 | case Intrinsic::x86_sse2_cvtsd2ss: | ||||||
3048 | case Intrinsic::x86_sse2_cvttsd2si64: | ||||||
3049 | case Intrinsic::x86_sse2_cvttsd2si: | ||||||
3050 | case Intrinsic::x86_sse_cvtss2si64: | ||||||
3051 | case Intrinsic::x86_sse_cvtss2si: | ||||||
3052 | case Intrinsic::x86_sse_cvttss2si64: | ||||||
3053 | case Intrinsic::x86_sse_cvttss2si: | ||||||
3054 | handleVectorConvertIntrinsic(I, 1); | ||||||
3055 | break; | ||||||
3056 | case Intrinsic::x86_sse_cvtps2pi: | ||||||
3057 | case Intrinsic::x86_sse_cvttps2pi: | ||||||
3058 | handleVectorConvertIntrinsic(I, 2); | ||||||
3059 | break; | ||||||
3060 | |||||||
3061 | case Intrinsic::x86_avx512_psll_w_512: | ||||||
3062 | case Intrinsic::x86_avx512_psll_d_512: | ||||||
3063 | case Intrinsic::x86_avx512_psll_q_512: | ||||||
3064 | case Intrinsic::x86_avx512_pslli_w_512: | ||||||
3065 | case Intrinsic::x86_avx512_pslli_d_512: | ||||||
3066 | case Intrinsic::x86_avx512_pslli_q_512: | ||||||
3067 | case Intrinsic::x86_avx512_psrl_w_512: | ||||||
3068 | case Intrinsic::x86_avx512_psrl_d_512: | ||||||
3069 | case Intrinsic::x86_avx512_psrl_q_512: | ||||||
3070 | case Intrinsic::x86_avx512_psra_w_512: | ||||||
3071 | case Intrinsic::x86_avx512_psra_d_512: | ||||||
3072 | case Intrinsic::x86_avx512_psra_q_512: | ||||||
3073 | case Intrinsic::x86_avx512_psrli_w_512: | ||||||
3074 | case Intrinsic::x86_avx512_psrli_d_512: | ||||||
3075 | case Intrinsic::x86_avx512_psrli_q_512: | ||||||
3076 | case Intrinsic::x86_avx512_psrai_w_512: | ||||||
3077 | case Intrinsic::x86_avx512_psrai_d_512: | ||||||
3078 | case Intrinsic::x86_avx512_psrai_q_512: | ||||||
3079 | case Intrinsic::x86_avx512_psra_q_256: | ||||||
3080 | case Intrinsic::x86_avx512_psra_q_128: | ||||||
3081 | case Intrinsic::x86_avx512_psrai_q_256: | ||||||
3082 | case Intrinsic::x86_avx512_psrai_q_128: | ||||||
3083 | case Intrinsic::x86_avx2_psll_w: | ||||||
3084 | case Intrinsic::x86_avx2_psll_d: | ||||||
3085 | case Intrinsic::x86_avx2_psll_q: | ||||||
3086 | case Intrinsic::x86_avx2_pslli_w: | ||||||
3087 | case Intrinsic::x86_avx2_pslli_d: | ||||||
3088 | case Intrinsic::x86_avx2_pslli_q: | ||||||
3089 | case Intrinsic::x86_avx2_psrl_w: | ||||||
3090 | case Intrinsic::x86_avx2_psrl_d: | ||||||
3091 | case Intrinsic::x86_avx2_psrl_q: | ||||||
3092 | case Intrinsic::x86_avx2_psra_w: | ||||||
3093 | case Intrinsic::x86_avx2_psra_d: | ||||||
3094 | case Intrinsic::x86_avx2_psrli_w: | ||||||
3095 | case Intrinsic::x86_avx2_psrli_d: | ||||||
3096 | case Intrinsic::x86_avx2_psrli_q: | ||||||
3097 | case Intrinsic::x86_avx2_psrai_w: | ||||||
3098 | case Intrinsic::x86_avx2_psrai_d: | ||||||
3099 | case Intrinsic::x86_sse2_psll_w: | ||||||
3100 | case Intrinsic::x86_sse2_psll_d: | ||||||
3101 | case Intrinsic::x86_sse2_psll_q: | ||||||
3102 | case Intrinsic::x86_sse2_pslli_w: | ||||||
3103 | case Intrinsic::x86_sse2_pslli_d: | ||||||
3104 | case Intrinsic::x86_sse2_pslli_q: | ||||||
3105 | case Intrinsic::x86_sse2_psrl_w: | ||||||
3106 | case Intrinsic::x86_sse2_psrl_d: | ||||||
3107 | case Intrinsic::x86_sse2_psrl_q: | ||||||
3108 | case Intrinsic::x86_sse2_psra_w: | ||||||
3109 | case Intrinsic::x86_sse2_psra_d: | ||||||
3110 | case Intrinsic::x86_sse2_psrli_w: | ||||||
3111 | case Intrinsic::x86_sse2_psrli_d: | ||||||
3112 | case Intrinsic::x86_sse2_psrli_q: | ||||||
3113 | case Intrinsic::x86_sse2_psrai_w: | ||||||
3114 | case Intrinsic::x86_sse2_psrai_d: | ||||||
3115 | case Intrinsic::x86_mmx_psll_w: | ||||||
3116 | case Intrinsic::x86_mmx_psll_d: | ||||||
3117 | case Intrinsic::x86_mmx_psll_q: | ||||||
3118 | case Intrinsic::x86_mmx_pslli_w: | ||||||
3119 | case Intrinsic::x86_mmx_pslli_d: | ||||||
3120 | case Intrinsic::x86_mmx_pslli_q: | ||||||
3121 | case Intrinsic::x86_mmx_psrl_w: | ||||||
3122 | case Intrinsic::x86_mmx_psrl_d: | ||||||
3123 | case Intrinsic::x86_mmx_psrl_q: | ||||||
3124 | case Intrinsic::x86_mmx_psra_w: | ||||||
3125 | case Intrinsic::x86_mmx_psra_d: | ||||||
3126 | case Intrinsic::x86_mmx_psrli_w: | ||||||
3127 | case Intrinsic::x86_mmx_psrli_d: | ||||||
3128 | case Intrinsic::x86_mmx_psrli_q: | ||||||
3129 | case Intrinsic::x86_mmx_psrai_w: | ||||||
3130 | case Intrinsic::x86_mmx_psrai_d: | ||||||
3131 | handleVectorShiftIntrinsic(I, /* Variable */ false); | ||||||
3132 | break; | ||||||
3133 | case Intrinsic::x86_avx2_psllv_d: | ||||||
3134 | case Intrinsic::x86_avx2_psllv_d_256: | ||||||
3135 | case Intrinsic::x86_avx512_psllv_d_512: | ||||||
3136 | case Intrinsic::x86_avx2_psllv_q: | ||||||
3137 | case Intrinsic::x86_avx2_psllv_q_256: | ||||||
3138 | case Intrinsic::x86_avx512_psllv_q_512: | ||||||
3139 | case Intrinsic::x86_avx2_psrlv_d: | ||||||
3140 | case Intrinsic::x86_avx2_psrlv_d_256: | ||||||
3141 | case Intrinsic::x86_avx512_psrlv_d_512: | ||||||
3142 | case Intrinsic::x86_avx2_psrlv_q: | ||||||
3143 | case Intrinsic::x86_avx2_psrlv_q_256: | ||||||
3144 | case Intrinsic::x86_avx512_psrlv_q_512: | ||||||
3145 | case Intrinsic::x86_avx2_psrav_d: | ||||||
3146 | case Intrinsic::x86_avx2_psrav_d_256: | ||||||
3147 | case Intrinsic::x86_avx512_psrav_d_512: | ||||||
3148 | case Intrinsic::x86_avx512_psrav_q_128: | ||||||
3149 | case Intrinsic::x86_avx512_psrav_q_256: | ||||||
3150 | case Intrinsic::x86_avx512_psrav_q_512: | ||||||
3151 | handleVectorShiftIntrinsic(I, /* Variable */ true); | ||||||
3152 | break; | ||||||
3153 | |||||||
3154 | case Intrinsic::x86_sse2_packsswb_128: | ||||||
3155 | case Intrinsic::x86_sse2_packssdw_128: | ||||||
3156 | case Intrinsic::x86_sse2_packuswb_128: | ||||||
3157 | case Intrinsic::x86_sse41_packusdw: | ||||||
3158 | case Intrinsic::x86_avx2_packsswb: | ||||||
3159 | case Intrinsic::x86_avx2_packssdw: | ||||||
3160 | case Intrinsic::x86_avx2_packuswb: | ||||||
3161 | case Intrinsic::x86_avx2_packusdw: | ||||||
3162 | handleVectorPackIntrinsic(I); | ||||||
3163 | break; | ||||||
3164 | |||||||
3165 | case Intrinsic::x86_mmx_packsswb: | ||||||
3166 | case Intrinsic::x86_mmx_packuswb: | ||||||
3167 | handleVectorPackIntrinsic(I, 16); | ||||||
3168 | break; | ||||||
3169 | |||||||
3170 | case Intrinsic::x86_mmx_packssdw: | ||||||
3171 | handleVectorPackIntrinsic(I, 32); | ||||||
3172 | break; | ||||||
3173 | |||||||
3174 | case Intrinsic::x86_mmx_psad_bw: | ||||||
3175 | case Intrinsic::x86_sse2_psad_bw: | ||||||
3176 | case Intrinsic::x86_avx2_psad_bw: | ||||||
3177 | handleVectorSadIntrinsic(I); | ||||||
3178 | break; | ||||||
3179 | |||||||
3180 | case Intrinsic::x86_sse2_pmadd_wd: | ||||||
3181 | case Intrinsic::x86_avx2_pmadd_wd: | ||||||
3182 | case Intrinsic::x86_ssse3_pmadd_ub_sw_128: | ||||||
3183 | case Intrinsic::x86_avx2_pmadd_ub_sw: | ||||||
3184 | handleVectorPmaddIntrinsic(I); | ||||||
3185 | break; | ||||||
3186 | |||||||
3187 | case Intrinsic::x86_ssse3_pmadd_ub_sw: | ||||||
3188 | handleVectorPmaddIntrinsic(I, 8); | ||||||
3189 | break; | ||||||
3190 | |||||||
3191 | case Intrinsic::x86_mmx_pmadd_wd: | ||||||
3192 | handleVectorPmaddIntrinsic(I, 16); | ||||||
3193 | break; | ||||||
3194 | |||||||
3195 | case Intrinsic::x86_sse_cmp_ss: | ||||||
3196 | case Intrinsic::x86_sse2_cmp_sd: | ||||||
3197 | case Intrinsic::x86_sse_comieq_ss: | ||||||
3198 | case Intrinsic::x86_sse_comilt_ss: | ||||||
3199 | case Intrinsic::x86_sse_comile_ss: | ||||||
3200 | case Intrinsic::x86_sse_comigt_ss: | ||||||
3201 | case Intrinsic::x86_sse_comige_ss: | ||||||
3202 | case Intrinsic::x86_sse_comineq_ss: | ||||||
3203 | case Intrinsic::x86_sse_ucomieq_ss: | ||||||
3204 | case Intrinsic::x86_sse_ucomilt_ss: | ||||||
3205 | case Intrinsic::x86_sse_ucomile_ss: | ||||||
3206 | case Intrinsic::x86_sse_ucomigt_ss: | ||||||
3207 | case Intrinsic::x86_sse_ucomige_ss: | ||||||
3208 | case Intrinsic::x86_sse_ucomineq_ss: | ||||||
3209 | case Intrinsic::x86_sse2_comieq_sd: | ||||||
3210 | case Intrinsic::x86_sse2_comilt_sd: | ||||||
3211 | case Intrinsic::x86_sse2_comile_sd: | ||||||
3212 | case Intrinsic::x86_sse2_comigt_sd: | ||||||
3213 | case Intrinsic::x86_sse2_comige_sd: | ||||||
3214 | case Intrinsic::x86_sse2_comineq_sd: | ||||||
3215 | case Intrinsic::x86_sse2_ucomieq_sd: | ||||||
3216 | case Intrinsic::x86_sse2_ucomilt_sd: | ||||||
3217 | case Intrinsic::x86_sse2_ucomile_sd: | ||||||
3218 | case Intrinsic::x86_sse2_ucomigt_sd: | ||||||
3219 | case Intrinsic::x86_sse2_ucomige_sd: | ||||||
3220 | case Intrinsic::x86_sse2_ucomineq_sd: | ||||||
3221 | handleVectorCompareScalarIntrinsic(I); | ||||||
3222 | break; | ||||||
3223 | |||||||
3224 | case Intrinsic::x86_sse_cmp_ps: | ||||||
3225 | case Intrinsic::x86_sse2_cmp_pd: | ||||||
3226 | // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function | ||||||
3227 | // generates reasonably looking IR that fails in the backend with "Do not | ||||||
3228 | // know how to split the result of this operator!". | ||||||
3229 | handleVectorComparePackedIntrinsic(I); | ||||||
3230 | break; | ||||||
3231 | |||||||
3232 | case Intrinsic::x86_bmi_bextr_32: | ||||||
3233 | case Intrinsic::x86_bmi_bextr_64: | ||||||
3234 | case Intrinsic::x86_bmi_bzhi_32: | ||||||
3235 | case Intrinsic::x86_bmi_bzhi_64: | ||||||
3236 | case Intrinsic::x86_bmi_pdep_32: | ||||||
3237 | case Intrinsic::x86_bmi_pdep_64: | ||||||
3238 | case Intrinsic::x86_bmi_pext_32: | ||||||
3239 | case Intrinsic::x86_bmi_pext_64: | ||||||
3240 | handleBmiIntrinsic(I); | ||||||
3241 | break; | ||||||
3242 | |||||||
3243 | case Intrinsic::is_constant: | ||||||
3244 | // The result of llvm.is.constant() is always defined. | ||||||
3245 | setShadow(&I, getCleanShadow(&I)); | ||||||
3246 | setOrigin(&I, getCleanOrigin()); | ||||||
3247 | break; | ||||||
3248 | |||||||
3249 | default: | ||||||
3250 | if (!handleUnknownIntrinsic(I)) | ||||||
3251 | visitInstruction(I); | ||||||
3252 | break; | ||||||
3253 | } | ||||||
3254 | } | ||||||
3255 | |||||||
3256 | void visitCallSite(CallSite CS) { | ||||||
3257 | Instruction &I = *CS.getInstruction(); | ||||||
3258 | assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0) : __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3258, __PRETTY_FUNCTION__)); | ||||||
3259 | assert((CS.isCall() || CS.isInvoke() || CS.isCallBr()) &&(((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && "Unknown type of CallSite" ) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke() || CS.isCallBr()) && \"Unknown type of CallSite\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3260, __PRETTY_FUNCTION__)) | ||||||
3260 | "Unknown type of CallSite")(((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && "Unknown type of CallSite" ) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke() || CS.isCallBr()) && \"Unknown type of CallSite\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3260, __PRETTY_FUNCTION__)); | ||||||
3261 | if (CS.isCallBr() || (CS.isCall() && cast<CallInst>(&I)->isInlineAsm())) { | ||||||
3262 | // For inline asm (either a call to asm function, or callbr instruction), | ||||||
3263 | // do the usual thing: check argument shadow and mark all outputs as | ||||||
3264 | // clean. Note that any side effects of the inline asm that are not | ||||||
3265 | // immediately visible in its constraints are not handled. | ||||||
3266 | if (ClHandleAsmConservative && MS.CompileKernel) | ||||||
3267 | visitAsmInstruction(I); | ||||||
3268 | else | ||||||
3269 | visitInstruction(I); | ||||||
3270 | return; | ||||||
3271 | } | ||||||
3272 | if (CS.isCall()) { | ||||||
3273 | CallInst *Call = cast<CallInst>(&I); | ||||||
3274 | assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere")((!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere" ) ? static_cast<void> (0) : __assert_fail ("!isa<IntrinsicInst>(&I) && \"intrinsics are handled elsewhere\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3274, __PRETTY_FUNCTION__)); | ||||||
3275 | |||||||
3276 | // We are going to insert code that relies on the fact that the callee | ||||||
3277 | // will become a non-readonly function after it is instrumented by us. To | ||||||
3278 | // prevent this code from being optimized out, mark that function | ||||||
3279 | // non-readonly in advance. | ||||||
3280 | if (Function *Func = Call->getCalledFunction()) { | ||||||
3281 | // Clear out readonly/readnone attributes. | ||||||
3282 | AttrBuilder B; | ||||||
3283 | B.addAttribute(Attribute::ReadOnly) | ||||||
3284 | .addAttribute(Attribute::ReadNone) | ||||||
3285 | .addAttribute(Attribute::WriteOnly) | ||||||
3286 | .addAttribute(Attribute::ArgMemOnly) | ||||||
3287 | .addAttribute(Attribute::Speculatable); | ||||||
3288 | Func->removeAttributes(AttributeList::FunctionIndex, B); | ||||||
3289 | } | ||||||
3290 | |||||||
3291 | maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI); | ||||||
3292 | } | ||||||
3293 | IRBuilder<> IRB(&I); | ||||||
3294 | |||||||
3295 | unsigned ArgOffset = 0; | ||||||
3296 | LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " CallSite: " << I << "\n"; } } while (false); | ||||||
3297 | for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); | ||||||
3298 | ArgIt != End; ++ArgIt) { | ||||||
3299 | Value *A = *ArgIt; | ||||||
3300 | unsigned i = ArgIt - CS.arg_begin(); | ||||||
3301 | if (!A->getType()->isSized()) { | ||||||
3302 | LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "Arg " << i << " is not sized: " << I << "\n"; } } while (false); | ||||||
3303 | continue; | ||||||
3304 | } | ||||||
3305 | unsigned Size = 0; | ||||||
3306 | Value *Store = nullptr; | ||||||
3307 | // Compute the Shadow for arg even if it is ByVal, because | ||||||
3308 | // in that case getShadow() will copy the actual arg shadow to | ||||||
3309 | // __msan_param_tls. | ||||||
3310 | Value *ArgShadow = getShadow(A); | ||||||
3311 | Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset); | ||||||
3312 | LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *Ado { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " Arg#" << i << ": " << *A << " Shadow: " << *ArgShadow << "\n"; } } while (false) | ||||||
3313 | << " Shadow: " << *ArgShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " Arg#" << i << ": " << *A << " Shadow: " << *ArgShadow << "\n"; } } while (false); | ||||||
3314 | bool ArgIsInitialized = false; | ||||||
3315 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
3316 | if (CS.paramHasAttr(i, Attribute::ByVal)) { | ||||||
3317 | assert(A->getType()->isPointerTy() &&((A->getType()->isPointerTy() && "ByVal argument is not a pointer!" ) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3318, __PRETTY_FUNCTION__)) | ||||||
3318 | "ByVal argument is not a pointer!")((A->getType()->isPointerTy() && "ByVal argument is not a pointer!" ) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3318, __PRETTY_FUNCTION__)); | ||||||
3319 | Size = DL.getTypeAllocSize(A->getType()->getPointerElementType()); | ||||||
3320 | if (ArgOffset + Size > kParamTLSSize) break; | ||||||
3321 | const MaybeAlign ParamAlignment(CS.getParamAlignment(i)); | ||||||
3322 | MaybeAlign Alignment = llvm::None; | ||||||
3323 | if (ParamAlignment) | ||||||
3324 | Alignment = std::min(*ParamAlignment, kShadowTLSAlignment); | ||||||
3325 | Value *AShadowPtr = | ||||||
3326 | getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment, | ||||||
3327 | /*isStore*/ false) | ||||||
3328 | .first; | ||||||
3329 | |||||||
3330 | Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr, | ||||||
3331 | Alignment, Size); | ||||||
3332 | // TODO(glider): need to copy origins. | ||||||
3333 | } else { | ||||||
3334 | Size = DL.getTypeAllocSize(A->getType()); | ||||||
3335 | if (ArgOffset + Size > kParamTLSSize) break; | ||||||
3336 | Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase, | ||||||
3337 | kShadowTLSAlignment.value()); | ||||||
3338 | Constant *Cst = dyn_cast<Constant>(ArgShadow); | ||||||
3339 | if (Cst && Cst->isNullValue()) ArgIsInitialized = true; | ||||||
3340 | } | ||||||
3341 | if (MS.TrackOrigins && !ArgIsInitialized) | ||||||
3342 | IRB.CreateStore(getOrigin(A), | ||||||
3343 | getOriginPtrForArgument(A, IRB, ArgOffset)); | ||||||
3344 | (void)Store; | ||||||
3345 | assert(Size != 0 && Store != nullptr)((Size != 0 && Store != nullptr) ? static_cast<void > (0) : __assert_fail ("Size != 0 && Store != nullptr" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3345, __PRETTY_FUNCTION__)); | ||||||
3346 | LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " Param:" << *Store << "\n"; } } while (false); | ||||||
3347 | ArgOffset += alignTo(Size, 8); | ||||||
3348 | } | ||||||
3349 | LLVM_DEBUG(dbgs() << " done with call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " done with call args\n"; } } while (false); | ||||||
3350 | |||||||
3351 | FunctionType *FT = CS.getFunctionType(); | ||||||
3352 | if (FT->isVarArg()) { | ||||||
3353 | VAHelper->visitCallSite(CS, IRB); | ||||||
3354 | } | ||||||
3355 | |||||||
3356 | // Now, get the shadow for the RetVal. | ||||||
3357 | if (!I.getType()->isSized()) return; | ||||||
3358 | // Don't emit the epilogue for musttail call returns. | ||||||
3359 | if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return; | ||||||
3360 | IRBuilder<> IRBBefore(&I); | ||||||
3361 | // Until we have full dynamic coverage, make sure the retval shadow is 0. | ||||||
3362 | Value *Base = getShadowPtrForRetval(&I, IRBBefore); | ||||||
3363 | IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, | ||||||
3364 | kShadowTLSAlignment.value()); | ||||||
3365 | BasicBlock::iterator NextInsn; | ||||||
3366 | if (CS.isCall()) { | ||||||
3367 | NextInsn = ++I.getIterator(); | ||||||
3368 | assert(NextInsn != I.getParent()->end())((NextInsn != I.getParent()->end()) ? static_cast<void> (0) : __assert_fail ("NextInsn != I.getParent()->end()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3368, __PRETTY_FUNCTION__)); | ||||||
3369 | } else { | ||||||
3370 | BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest(); | ||||||
3371 | if (!NormalDest->getSinglePredecessor()) { | ||||||
3372 | // FIXME: this case is tricky, so we are just conservative here. | ||||||
3373 | // Perhaps we need to split the edge between this BB and NormalDest, | ||||||
3374 | // but a naive attempt to use SplitEdge leads to a crash. | ||||||
3375 | setShadow(&I, getCleanShadow(&I)); | ||||||
3376 | setOrigin(&I, getCleanOrigin()); | ||||||
3377 | return; | ||||||
3378 | } | ||||||
3379 | // FIXME: NextInsn is likely in a basic block that has not been visited yet. | ||||||
3380 | // Anything inserted there will be instrumented by MSan later! | ||||||
3381 | NextInsn = NormalDest->getFirstInsertionPt(); | ||||||
3382 | assert(NextInsn != NormalDest->end() &&((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load" ) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3383, __PRETTY_FUNCTION__)) | ||||||
3383 | "Could not find insertion point for retval shadow load")((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load" ) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3383, __PRETTY_FUNCTION__)); | ||||||
3384 | } | ||||||
3385 | IRBuilder<> IRBAfter(&*NextInsn); | ||||||
3386 | Value *RetvalShadow = IRBAfter.CreateAlignedLoad( | ||||||
3387 | getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter), | ||||||
3388 | kShadowTLSAlignment.value(), "_msret"); | ||||||
3389 | setShadow(&I, RetvalShadow); | ||||||
3390 | if (MS.TrackOrigins) | ||||||
3391 | setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy, | ||||||
3392 | getOriginPtrForRetval(IRBAfter))); | ||||||
3393 | } | ||||||
3394 | |||||||
3395 | bool isAMustTailRetVal(Value *RetVal) { | ||||||
3396 | if (auto *I = dyn_cast<BitCastInst>(RetVal)) { | ||||||
3397 | RetVal = I->getOperand(0); | ||||||
3398 | } | ||||||
3399 | if (auto *I = dyn_cast<CallInst>(RetVal)) { | ||||||
3400 | return I->isMustTailCall(); | ||||||
3401 | } | ||||||
3402 | return false; | ||||||
3403 | } | ||||||
3404 | |||||||
3405 | void visitReturnInst(ReturnInst &I) { | ||||||
3406 | IRBuilder<> IRB(&I); | ||||||
3407 | Value *RetVal = I.getReturnValue(); | ||||||
3408 | if (!RetVal) return; | ||||||
3409 | // Don't emit the epilogue for musttail call returns. | ||||||
3410 | if (isAMustTailRetVal(RetVal)) return; | ||||||
3411 | Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB); | ||||||
3412 | if (CheckReturnValue) { | ||||||
3413 | insertShadowCheck(RetVal, &I); | ||||||
3414 | Value *Shadow = getCleanShadow(RetVal); | ||||||
3415 | IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment.value()); | ||||||
3416 | } else { | ||||||
3417 | Value *Shadow = getShadow(RetVal); | ||||||
3418 | IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment.value()); | ||||||
3419 | if (MS.TrackOrigins) | ||||||
3420 | IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB)); | ||||||
3421 | } | ||||||
3422 | } | ||||||
3423 | |||||||
3424 | void visitPHINode(PHINode &I) { | ||||||
3425 | IRBuilder<> IRB(&I); | ||||||
3426 | if (!PropagateShadow) { | ||||||
3427 | setShadow(&I, getCleanShadow(&I)); | ||||||
3428 | setOrigin(&I, getCleanOrigin()); | ||||||
3429 | return; | ||||||
3430 | } | ||||||
3431 | |||||||
3432 | ShadowPHINodes.push_back(&I); | ||||||
3433 | setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(), | ||||||
3434 | "_msphi_s")); | ||||||
3435 | if (MS.TrackOrigins) | ||||||
3436 | setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), | ||||||
3437 | "_msphi_o")); | ||||||
3438 | } | ||||||
3439 | |||||||
3440 | Value *getLocalVarDescription(AllocaInst &I) { | ||||||
3441 | SmallString<2048> StackDescriptionStorage; | ||||||
3442 | raw_svector_ostream StackDescription(StackDescriptionStorage); | ||||||
3443 | // We create a string with a description of the stack allocation and | ||||||
3444 | // pass it into __msan_set_alloca_origin. | ||||||
3445 | // It will be printed by the run-time if stack-originated UMR is found. | ||||||
3446 | // The first 4 bytes of the string are set to '----' and will be replaced | ||||||
3447 | // by __msan_va_arg_overflow_size_tls at the first call. | ||||||
3448 | StackDescription << "----" << I.getName() << "@" << F.getName(); | ||||||
3449 | return createPrivateNonConstGlobalForString(*F.getParent(), | ||||||
3450 | StackDescription.str()); | ||||||
3451 | } | ||||||
3452 | |||||||
3453 | void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { | ||||||
3454 | if (PoisonStack && ClPoisonStackWithCall) { | ||||||
3455 | IRB.CreateCall(MS.MsanPoisonStackFn, | ||||||
3456 | {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); | ||||||
3457 | } else { | ||||||
3458 | Value *ShadowBase, *OriginBase; | ||||||
3459 | std::tie(ShadowBase, OriginBase) = getShadowOriginPtr( | ||||||
3460 | &I, IRB, IRB.getInt8Ty(), Align::None(), /*isStore*/ true); | ||||||
3461 | |||||||
3462 | Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0); | ||||||
3463 | IRB.CreateMemSet(ShadowBase, PoisonValue, Len, | ||||||
3464 | MaybeAlign(I.getAlignment())); | ||||||
3465 | } | ||||||
3466 | |||||||
3467 | if (PoisonStack && MS.TrackOrigins) { | ||||||
3468 | Value *Descr = getLocalVarDescription(I); | ||||||
3469 | IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn, | ||||||
3470 | {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, | ||||||
3471 | IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()), | ||||||
3472 | IRB.CreatePointerCast(&F, MS.IntptrTy)}); | ||||||
3473 | } | ||||||
3474 | } | ||||||
3475 | |||||||
3476 | void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) { | ||||||
3477 | Value *Descr = getLocalVarDescription(I); | ||||||
3478 | if (PoisonStack) { | ||||||
3479 | IRB.CreateCall(MS.MsanPoisonAllocaFn, | ||||||
3480 | {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len, | ||||||
3481 | IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())}); | ||||||
3482 | } else { | ||||||
3483 | IRB.CreateCall(MS.MsanUnpoisonAllocaFn, | ||||||
3484 | {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len}); | ||||||
3485 | } | ||||||
3486 | } | ||||||
3487 | |||||||
3488 | void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) { | ||||||
3489 | if (!InsPoint) | ||||||
3490 | InsPoint = &I; | ||||||
3491 | IRBuilder<> IRB(InsPoint->getNextNode()); | ||||||
3492 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
3493 | uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType()); | ||||||
3494 | Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize); | ||||||
3495 | if (I.isArrayAllocation()) | ||||||
3496 | Len = IRB.CreateMul(Len, I.getArraySize()); | ||||||
3497 | |||||||
3498 | if (MS.CompileKernel) | ||||||
3499 | poisonAllocaKmsan(I, IRB, Len); | ||||||
3500 | else | ||||||
3501 | poisonAllocaUserspace(I, IRB, Len); | ||||||
3502 | } | ||||||
3503 | |||||||
3504 | void visitAllocaInst(AllocaInst &I) { | ||||||
3505 | setShadow(&I, getCleanShadow(&I)); | ||||||
3506 | setOrigin(&I, getCleanOrigin()); | ||||||
3507 | // We'll get to this alloca later unless it's poisoned at the corresponding | ||||||
3508 | // llvm.lifetime.start. | ||||||
3509 | AllocaSet.insert(&I); | ||||||
3510 | } | ||||||
3511 | |||||||
3512 | void visitSelectInst(SelectInst& I) { | ||||||
3513 | IRBuilder<> IRB(&I); | ||||||
3514 | // a = select b, c, d | ||||||
3515 | Value *B = I.getCondition(); | ||||||
3516 | Value *C = I.getTrueValue(); | ||||||
3517 | Value *D = I.getFalseValue(); | ||||||
3518 | Value *Sb = getShadow(B); | ||||||
3519 | Value *Sc = getShadow(C); | ||||||
3520 | Value *Sd = getShadow(D); | ||||||
3521 | |||||||
3522 | // Result shadow if condition shadow is 0. | ||||||
3523 | Value *Sa0 = IRB.CreateSelect(B, Sc, Sd); | ||||||
3524 | Value *Sa1; | ||||||
3525 | if (I.getType()->isAggregateType()) { | ||||||
3526 | // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do | ||||||
3527 | // an extra "select". This results in much more compact IR. | ||||||
3528 | // Sa = select Sb, poisoned, (select b, Sc, Sd) | ||||||
3529 | Sa1 = getPoisonedShadow(getShadowTy(I.getType())); | ||||||
3530 | } else { | ||||||
3531 | // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ] | ||||||
3532 | // If Sb (condition is poisoned), look for bits in c and d that are equal | ||||||
3533 | // and both unpoisoned. | ||||||
3534 | // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd. | ||||||
3535 | |||||||
3536 | // Cast arguments to shadow-compatible type. | ||||||
3537 | C = CreateAppToShadowCast(IRB, C); | ||||||
3538 | D = CreateAppToShadowCast(IRB, D); | ||||||
3539 | |||||||
3540 | // Result shadow if condition shadow is 1. | ||||||
3541 | Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd}); | ||||||
3542 | } | ||||||
3543 | Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select"); | ||||||
3544 | setShadow(&I, Sa); | ||||||
3545 | if (MS.TrackOrigins) { | ||||||
3546 | // Origins are always i32, so any vector conditions must be flattened. | ||||||
3547 | // FIXME: consider tracking vector origins for app vectors? | ||||||
3548 | if (B->getType()->isVectorTy()) { | ||||||
3549 | Type *FlatTy = getShadowTyNoVec(B->getType()); | ||||||
3550 | B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy), | ||||||
3551 | ConstantInt::getNullValue(FlatTy)); | ||||||
3552 | Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy), | ||||||
3553 | ConstantInt::getNullValue(FlatTy)); | ||||||
3554 | } | ||||||
3555 | // a = select b, c, d | ||||||
3556 | // Oa = Sb ? Ob : (b ? Oc : Od) | ||||||
3557 | setOrigin( | ||||||
3558 | &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()), | ||||||
3559 | IRB.CreateSelect(B, getOrigin(I.getTrueValue()), | ||||||
3560 | getOrigin(I.getFalseValue())))); | ||||||
3561 | } | ||||||
3562 | } | ||||||
3563 | |||||||
3564 | void visitLandingPadInst(LandingPadInst &I) { | ||||||
3565 | // Do nothing. | ||||||
3566 | // See https://github.com/google/sanitizers/issues/504 | ||||||
3567 | setShadow(&I, getCleanShadow(&I)); | ||||||
3568 | setOrigin(&I, getCleanOrigin()); | ||||||
3569 | } | ||||||
3570 | |||||||
3571 | void visitCatchSwitchInst(CatchSwitchInst &I) { | ||||||
3572 | setShadow(&I, getCleanShadow(&I)); | ||||||
3573 | setOrigin(&I, getCleanOrigin()); | ||||||
3574 | } | ||||||
3575 | |||||||
3576 | void visitFuncletPadInst(FuncletPadInst &I) { | ||||||
3577 | setShadow(&I, getCleanShadow(&I)); | ||||||
3578 | setOrigin(&I, getCleanOrigin()); | ||||||
3579 | } | ||||||
3580 | |||||||
3581 | void visitGetElementPtrInst(GetElementPtrInst &I) { | ||||||
3582 | handleShadowOr(I); | ||||||
3583 | } | ||||||
3584 | |||||||
3585 | void visitExtractValueInst(ExtractValueInst &I) { | ||||||
3586 | IRBuilder<> IRB(&I); | ||||||
3587 | Value *Agg = I.getAggregateOperand(); | ||||||
3588 | LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "ExtractValue: " << I << "\n"; } } while (false); | ||||||
3589 | Value *AggShadow = getShadow(Agg); | ||||||
3590 | LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " AggShadow: " << *AggShadow << "\n"; } } while (false); | ||||||
3591 | Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices()); | ||||||
3592 | LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " ResShadow: " << *ResShadow << "\n"; } } while (false); | ||||||
3593 | setShadow(&I, ResShadow); | ||||||
3594 | setOriginForNaryOp(I); | ||||||
3595 | } | ||||||
3596 | |||||||
3597 | void visitInsertValueInst(InsertValueInst &I) { | ||||||
3598 | IRBuilder<> IRB(&I); | ||||||
3599 | LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "InsertValue: " << I << "\n"; } } while (false); | ||||||
3600 | Value *AggShadow = getShadow(I.getAggregateOperand()); | ||||||
3601 | Value *InsShadow = getShadow(I.getInsertedValueOperand()); | ||||||
3602 | LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " AggShadow: " << *AggShadow << "\n"; } } while (false); | ||||||
3603 | LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " InsShadow: " << *InsShadow << "\n"; } } while (false); | ||||||
3604 | Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices()); | ||||||
3605 | LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << " Res: " << *Res << "\n"; } } while (false); | ||||||
3606 | setShadow(&I, Res); | ||||||
3607 | setOriginForNaryOp(I); | ||||||
3608 | } | ||||||
3609 | |||||||
3610 | void dumpInst(Instruction &I) { | ||||||
3611 | if (CallInst *CI = dyn_cast<CallInst>(&I)) { | ||||||
3612 | errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n"; | ||||||
3613 | } else { | ||||||
3614 | errs() << "ZZZ " << I.getOpcodeName() << "\n"; | ||||||
3615 | } | ||||||
3616 | errs() << "QQQ " << I << "\n"; | ||||||
3617 | } | ||||||
3618 | |||||||
3619 | void visitResumeInst(ResumeInst &I) { | ||||||
3620 | LLVM_DEBUG(dbgs() << "Resume: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "Resume: " << I << "\n" ; } } while (false); | ||||||
3621 | // Nothing to do here. | ||||||
3622 | } | ||||||
3623 | |||||||
3624 | void visitCleanupReturnInst(CleanupReturnInst &CRI) { | ||||||
3625 | LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "CleanupReturn: " << CRI << "\n"; } } while (false); | ||||||
3626 | // Nothing to do here. | ||||||
3627 | } | ||||||
3628 | |||||||
3629 | void visitCatchReturnInst(CatchReturnInst &CRI) { | ||||||
3630 | LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "CatchReturn: " << CRI << "\n"; } } while (false); | ||||||
3631 | // Nothing to do here. | ||||||
3632 | } | ||||||
3633 | |||||||
3634 | void instrumentAsmArgument(Value *Operand, Instruction &I, IRBuilder<> &IRB, | ||||||
3635 | const DataLayout &DL, bool isOutput) { | ||||||
3636 | // For each assembly argument, we check its value for being initialized. | ||||||
3637 | // If the argument is a pointer, we assume it points to a single element | ||||||
3638 | // of the corresponding type (or to a 8-byte word, if the type is unsized). | ||||||
3639 | // Each such pointer is instrumented with a call to the runtime library. | ||||||
3640 | Type *OpType = Operand->getType(); | ||||||
3641 | // Check the operand value itself. | ||||||
3642 | insertShadowCheck(Operand, &I); | ||||||
3643 | if (!OpType->isPointerTy() || !isOutput) { | ||||||
3644 | assert(!isOutput)((!isOutput) ? static_cast<void> (0) : __assert_fail ("!isOutput" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3644, __PRETTY_FUNCTION__)); | ||||||
3645 | return; | ||||||
3646 | } | ||||||
3647 | Type *ElType = OpType->getPointerElementType(); | ||||||
3648 | if (!ElType->isSized()) | ||||||
3649 | return; | ||||||
3650 | int Size = DL.getTypeStoreSize(ElType); | ||||||
3651 | Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy()); | ||||||
3652 | Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size); | ||||||
3653 | IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal}); | ||||||
3654 | } | ||||||
3655 | |||||||
3656 | /// Get the number of output arguments returned by pointers. | ||||||
3657 | int getNumOutputArgs(InlineAsm *IA, CallBase *CB) { | ||||||
3658 | int NumRetOutputs = 0; | ||||||
3659 | int NumOutputs = 0; | ||||||
3660 | Type *RetTy = cast<Value>(CB)->getType(); | ||||||
3661 | if (!RetTy->isVoidTy()) { | ||||||
3662 | // Register outputs are returned via the CallInst return value. | ||||||
3663 | auto *ST = dyn_cast<StructType>(RetTy); | ||||||
3664 | if (ST) | ||||||
3665 | NumRetOutputs = ST->getNumElements(); | ||||||
3666 | else | ||||||
3667 | NumRetOutputs = 1; | ||||||
3668 | } | ||||||
3669 | InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints(); | ||||||
3670 | for (size_t i = 0, n = Constraints.size(); i < n; i++) { | ||||||
3671 | InlineAsm::ConstraintInfo Info = Constraints[i]; | ||||||
3672 | switch (Info.Type) { | ||||||
3673 | case InlineAsm::isOutput: | ||||||
3674 | NumOutputs++; | ||||||
3675 | break; | ||||||
3676 | default: | ||||||
3677 | break; | ||||||
3678 | } | ||||||
3679 | } | ||||||
3680 | return NumOutputs - NumRetOutputs; | ||||||
3681 | } | ||||||
3682 | |||||||
3683 | void visitAsmInstruction(Instruction &I) { | ||||||
3684 | // Conservative inline assembly handling: check for poisoned shadow of | ||||||
3685 | // asm() arguments, then unpoison the result and all the memory locations | ||||||
3686 | // pointed to by those arguments. | ||||||
3687 | // An inline asm() statement in C++ contains lists of input and output | ||||||
3688 | // arguments used by the assembly code. These are mapped to operands of the | ||||||
3689 | // CallInst as follows: | ||||||
3690 | // - nR register outputs ("=r) are returned by value in a single structure | ||||||
3691 | // (SSA value of the CallInst); | ||||||
3692 | // - nO other outputs ("=m" and others) are returned by pointer as first | ||||||
3693 | // nO operands of the CallInst; | ||||||
3694 | // - nI inputs ("r", "m" and others) are passed to CallInst as the | ||||||
3695 | // remaining nI operands. | ||||||
3696 | // The total number of asm() arguments in the source is nR+nO+nI, and the | ||||||
3697 | // corresponding CallInst has nO+nI+1 operands (the last operand is the | ||||||
3698 | // function to be called). | ||||||
3699 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
3700 | CallBase *CB = cast<CallBase>(&I); | ||||||
3701 | IRBuilder<> IRB(&I); | ||||||
3702 | InlineAsm *IA = cast<InlineAsm>(CB->getCalledValue()); | ||||||
3703 | int OutputArgs = getNumOutputArgs(IA, CB); | ||||||
3704 | // The last operand of a CallInst is the function itself. | ||||||
3705 | int NumOperands = CB->getNumOperands() - 1; | ||||||
3706 | |||||||
3707 | // Check input arguments. Doing so before unpoisoning output arguments, so | ||||||
3708 | // that we won't overwrite uninit values before checking them. | ||||||
3709 | for (int i = OutputArgs; i < NumOperands; i++) { | ||||||
3710 | Value *Operand = CB->getOperand(i); | ||||||
3711 | instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false); | ||||||
3712 | } | ||||||
3713 | // Unpoison output arguments. This must happen before the actual InlineAsm | ||||||
3714 | // call, so that the shadow for memory published in the asm() statement | ||||||
3715 | // remains valid. | ||||||
3716 | for (int i = 0; i < OutputArgs; i++) { | ||||||
3717 | Value *Operand = CB->getOperand(i); | ||||||
3718 | instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true); | ||||||
3719 | } | ||||||
3720 | |||||||
3721 | setShadow(&I, getCleanShadow(&I)); | ||||||
3722 | setOrigin(&I, getCleanOrigin()); | ||||||
3723 | } | ||||||
3724 | |||||||
3725 | void visitInstruction(Instruction &I) { | ||||||
3726 | // Everything else: stop propagating and check for poisoned shadow. | ||||||
3727 | if (ClDumpStrictInstructions) | ||||||
3728 | dumpInst(I); | ||||||
3729 | LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("msan")) { dbgs() << "DEFAULT: " << I << "\n" ; } } while (false); | ||||||
3730 | for (size_t i = 0, n = I.getNumOperands(); i < n; i++) { | ||||||
3731 | Value *Operand = I.getOperand(i); | ||||||
3732 | if (Operand->getType()->isSized()) | ||||||
3733 | insertShadowCheck(Operand, &I); | ||||||
3734 | } | ||||||
3735 | setShadow(&I, getCleanShadow(&I)); | ||||||
3736 | setOrigin(&I, getCleanOrigin()); | ||||||
3737 | } | ||||||
3738 | }; | ||||||
3739 | |||||||
3740 | /// AMD64-specific implementation of VarArgHelper. | ||||||
3741 | struct VarArgAMD64Helper : public VarArgHelper { | ||||||
3742 | // An unfortunate workaround for asymmetric lowering of va_arg stuff. | ||||||
3743 | // See a comment in visitCallSite for more details. | ||||||
3744 | static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7 | ||||||
3745 | static const unsigned AMD64FpEndOffsetSSE = 176; | ||||||
3746 | // If SSE is disabled, fp_offset in va_list is zero. | ||||||
3747 | static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset; | ||||||
3748 | |||||||
3749 | unsigned AMD64FpEndOffset; | ||||||
3750 | Function &F; | ||||||
3751 | MemorySanitizer &MS; | ||||||
3752 | MemorySanitizerVisitor &MSV; | ||||||
3753 | Value *VAArgTLSCopy = nullptr; | ||||||
3754 | Value *VAArgTLSOriginCopy = nullptr; | ||||||
3755 | Value *VAArgOverflowSize = nullptr; | ||||||
3756 | |||||||
3757 | SmallVector<CallInst*, 16> VAStartInstrumentationList; | ||||||
3758 | |||||||
3759 | enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory }; | ||||||
3760 | |||||||
3761 | VarArgAMD64Helper(Function &F, MemorySanitizer &MS, | ||||||
3762 | MemorySanitizerVisitor &MSV) | ||||||
3763 | : F(F), MS(MS), MSV(MSV) { | ||||||
3764 | AMD64FpEndOffset = AMD64FpEndOffsetSSE; | ||||||
3765 | for (const auto &Attr : F.getAttributes().getFnAttributes()) { | ||||||
3766 | if (Attr.isStringAttribute() && | ||||||
3767 | (Attr.getKindAsString() == "target-features")) { | ||||||
3768 | if (Attr.getValueAsString().contains("-sse")) | ||||||
3769 | AMD64FpEndOffset = AMD64FpEndOffsetNoSSE; | ||||||
3770 | break; | ||||||
3771 | } | ||||||
3772 | } | ||||||
3773 | } | ||||||
3774 | |||||||
3775 | ArgKind classifyArgument(Value* arg) { | ||||||
3776 | // A very rough approximation of X86_64 argument classification rules. | ||||||
3777 | Type *T = arg->getType(); | ||||||
3778 | if (T->isFPOrFPVectorTy() || T->isX86_MMXTy()) | ||||||
3779 | return AK_FloatingPoint; | ||||||
3780 | if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64) | ||||||
3781 | return AK_GeneralPurpose; | ||||||
3782 | if (T->isPointerTy()) | ||||||
3783 | return AK_GeneralPurpose; | ||||||
3784 | return AK_Memory; | ||||||
3785 | } | ||||||
3786 | |||||||
3787 | // For VarArg functions, store the argument shadow in an ABI-specific format | ||||||
3788 | // that corresponds to va_list layout. | ||||||
3789 | // We do this because Clang lowers va_arg in the frontend, and this pass | ||||||
3790 | // only sees the low level code that deals with va_list internals. | ||||||
3791 | // A much easier alternative (provided that Clang emits va_arg instructions) | ||||||
3792 | // would have been to associate each live instance of va_list with a copy of | ||||||
3793 | // MSanParamTLS, and extract shadow on va_arg() call in the argument list | ||||||
3794 | // order. | ||||||
3795 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { | ||||||
3796 | unsigned GpOffset = 0; | ||||||
3797 | unsigned FpOffset = AMD64GpEndOffset; | ||||||
3798 | unsigned OverflowOffset = AMD64FpEndOffset; | ||||||
3799 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
3800 | for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); | ||||||
3801 | ArgIt != End; ++ArgIt) { | ||||||
3802 | Value *A = *ArgIt; | ||||||
3803 | unsigned ArgNo = CS.getArgumentNo(ArgIt); | ||||||
3804 | bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams(); | ||||||
3805 | bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal); | ||||||
3806 | if (IsByVal) { | ||||||
3807 | // ByVal arguments always go to the overflow area. | ||||||
3808 | // Fixed arguments passed through the overflow area will be stepped | ||||||
3809 | // over by va_start, so don't count them towards the offset. | ||||||
3810 | if (IsFixed) | ||||||
3811 | continue; | ||||||
3812 | assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3812, __PRETTY_FUNCTION__)); | ||||||
3813 | Type *RealTy = A->getType()->getPointerElementType(); | ||||||
3814 | uint64_t ArgSize = DL.getTypeAllocSize(RealTy); | ||||||
3815 | Value *ShadowBase = getShadowPtrForVAArgument( | ||||||
3816 | RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8)); | ||||||
3817 | Value *OriginBase = nullptr; | ||||||
3818 | if (MS.TrackOrigins) | ||||||
3819 | OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset); | ||||||
3820 | OverflowOffset += alignTo(ArgSize, 8); | ||||||
3821 | if (!ShadowBase) | ||||||
3822 | continue; | ||||||
3823 | Value *ShadowPtr, *OriginPtr; | ||||||
3824 | std::tie(ShadowPtr, OriginPtr) = | ||||||
3825 | MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment, | ||||||
3826 | /*isStore*/ false); | ||||||
3827 | |||||||
3828 | IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr, | ||||||
3829 | kShadowTLSAlignment, ArgSize); | ||||||
3830 | if (MS.TrackOrigins) | ||||||
3831 | IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr, | ||||||
3832 | kShadowTLSAlignment, ArgSize); | ||||||
3833 | } else { | ||||||
3834 | ArgKind AK = classifyArgument(A); | ||||||
3835 | if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset) | ||||||
3836 | AK = AK_Memory; | ||||||
3837 | if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset) | ||||||
3838 | AK = AK_Memory; | ||||||
3839 | Value *ShadowBase, *OriginBase = nullptr; | ||||||
3840 | switch (AK) { | ||||||
3841 | case AK_GeneralPurpose: | ||||||
3842 | ShadowBase = | ||||||
3843 | getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8); | ||||||
3844 | if (MS.TrackOrigins) | ||||||
3845 | OriginBase = | ||||||
3846 | getOriginPtrForVAArgument(A->getType(), IRB, GpOffset); | ||||||
3847 | GpOffset += 8; | ||||||
3848 | break; | ||||||
3849 | case AK_FloatingPoint: | ||||||
3850 | ShadowBase = | ||||||
3851 | getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16); | ||||||
3852 | if (MS.TrackOrigins) | ||||||
3853 | OriginBase = | ||||||
3854 | getOriginPtrForVAArgument(A->getType(), IRB, FpOffset); | ||||||
3855 | FpOffset += 16; | ||||||
3856 | break; | ||||||
3857 | case AK_Memory: | ||||||
3858 | if (IsFixed) | ||||||
3859 | continue; | ||||||
3860 | uint64_t ArgSize = DL.getTypeAllocSize(A->getType()); | ||||||
3861 | ShadowBase = | ||||||
3862 | getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8); | ||||||
3863 | if (MS.TrackOrigins) | ||||||
3864 | OriginBase = | ||||||
3865 | getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset); | ||||||
3866 | OverflowOffset += alignTo(ArgSize, 8); | ||||||
3867 | } | ||||||
3868 | // Take fixed arguments into account for GpOffset and FpOffset, | ||||||
3869 | // but don't actually store shadows for them. | ||||||
3870 | // TODO(glider): don't call get*PtrForVAArgument() for them. | ||||||
3871 | if (IsFixed) | ||||||
3872 | continue; | ||||||
3873 | if (!ShadowBase) | ||||||
3874 | continue; | ||||||
3875 | Value *Shadow = MSV.getShadow(A); | ||||||
3876 | IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment.value()); | ||||||
3877 | if (MS.TrackOrigins) { | ||||||
3878 | Value *Origin = MSV.getOrigin(A); | ||||||
3879 | unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType()); | ||||||
3880 | MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize, | ||||||
3881 | std::max(kShadowTLSAlignment, kMinOriginAlignment)); | ||||||
3882 | } | ||||||
3883 | } | ||||||
3884 | } | ||||||
3885 | Constant *OverflowSize = | ||||||
3886 | ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset); | ||||||
3887 | IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); | ||||||
3888 | } | ||||||
3889 | |||||||
3890 | /// Compute the shadow address for a given va_arg. | ||||||
3891 | Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, | ||||||
3892 | unsigned ArgOffset, unsigned ArgSize) { | ||||||
3893 | // Make sure we don't overflow __msan_va_arg_tls. | ||||||
3894 | if (ArgOffset + ArgSize > kParamTLSSize) | ||||||
3895 | return nullptr; | ||||||
3896 | Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); | ||||||
3897 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
3898 | return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), | ||||||
3899 | "_msarg_va_s"); | ||||||
3900 | } | ||||||
3901 | |||||||
3902 | /// Compute the origin address for a given va_arg. | ||||||
3903 | Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) { | ||||||
3904 | Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy); | ||||||
3905 | // getOriginPtrForVAArgument() is always called after | ||||||
3906 | // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never | ||||||
3907 | // overflow. | ||||||
3908 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
3909 | return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0), | ||||||
3910 | "_msarg_va_o"); | ||||||
3911 | } | ||||||
3912 | |||||||
3913 | void unpoisonVAListTagForInst(IntrinsicInst &I) { | ||||||
3914 | IRBuilder<> IRB(&I); | ||||||
3915 | Value *VAListTag = I.getArgOperand(0); | ||||||
3916 | Value *ShadowPtr, *OriginPtr; | ||||||
3917 | const Align Alignment = Align(8); | ||||||
3918 | std::tie(ShadowPtr, OriginPtr) = | ||||||
3919 | MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment, | ||||||
3920 | /*isStore*/ true); | ||||||
3921 | |||||||
3922 | // Unpoison the whole __va_list_tag. | ||||||
3923 | // FIXME: magic ABI constants. | ||||||
3924 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
3925 | /* size */ 24, Alignment, false); | ||||||
3926 | // We shouldn't need to zero out the origins, as they're only checked for | ||||||
3927 | // nonzero shadow. | ||||||
3928 | } | ||||||
3929 | |||||||
3930 | void visitVAStartInst(VAStartInst &I) override { | ||||||
3931 | if (F.getCallingConv() == CallingConv::Win64) | ||||||
3932 | return; | ||||||
3933 | VAStartInstrumentationList.push_back(&I); | ||||||
3934 | unpoisonVAListTagForInst(I); | ||||||
3935 | } | ||||||
3936 | |||||||
3937 | void visitVACopyInst(VACopyInst &I) override { | ||||||
3938 | if (F.getCallingConv() == CallingConv::Win64) return; | ||||||
3939 | unpoisonVAListTagForInst(I); | ||||||
3940 | } | ||||||
3941 | |||||||
3942 | void finalizeInstrumentation() override { | ||||||
3943 | assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3944, __PRETTY_FUNCTION__)) | ||||||
3944 | "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 3944, __PRETTY_FUNCTION__)); | ||||||
3945 | if (!VAStartInstrumentationList.empty()) { | ||||||
3946 | // If there is a va_start in this function, make a backup copy of | ||||||
3947 | // va_arg_tls somewhere in the function entry block. | ||||||
3948 | IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); | ||||||
3949 | VAArgOverflowSize = | ||||||
3950 | IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); | ||||||
3951 | Value *CopySize = | ||||||
3952 | IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), | ||||||
3953 | VAArgOverflowSize); | ||||||
3954 | VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); | ||||||
3955 | IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); | ||||||
3956 | if (MS.TrackOrigins) { | ||||||
3957 | VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); | ||||||
3958 | IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS, | ||||||
3959 | Align(8), CopySize); | ||||||
3960 | } | ||||||
3961 | } | ||||||
3962 | |||||||
3963 | // Instrument va_start. | ||||||
3964 | // Copy va_list shadow from the backup copy of the TLS contents. | ||||||
3965 | for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) { | ||||||
3966 | CallInst *OrigInst = VAStartInstrumentationList[i]; | ||||||
3967 | IRBuilder<> IRB(OrigInst->getNextNode()); | ||||||
3968 | Value *VAListTag = OrigInst->getArgOperand(0); | ||||||
3969 | |||||||
3970 | Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); | ||||||
3971 | Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr( | ||||||
3972 | IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | ||||||
3973 | ConstantInt::get(MS.IntptrTy, 16)), | ||||||
3974 | PointerType::get(RegSaveAreaPtrTy, 0)); | ||||||
3975 | Value *RegSaveAreaPtr = | ||||||
3976 | IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); | ||||||
3977 | Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; | ||||||
3978 | const Align Alignment = Align(16); | ||||||
3979 | std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = | ||||||
3980 | MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
3981 | Alignment, /*isStore*/ true); | ||||||
3982 | IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, | ||||||
3983 | AMD64FpEndOffset); | ||||||
3984 | if (MS.TrackOrigins) | ||||||
3985 | IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy, | ||||||
3986 | Alignment, AMD64FpEndOffset); | ||||||
3987 | Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C); | ||||||
3988 | Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr( | ||||||
3989 | IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | ||||||
3990 | ConstantInt::get(MS.IntptrTy, 8)), | ||||||
3991 | PointerType::get(OverflowArgAreaPtrTy, 0)); | ||||||
3992 | Value *OverflowArgAreaPtr = | ||||||
3993 | IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr); | ||||||
3994 | Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr; | ||||||
3995 | std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) = | ||||||
3996 | MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
3997 | Alignment, /*isStore*/ true); | ||||||
3998 | Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy, | ||||||
3999 | AMD64FpEndOffset); | ||||||
4000 | IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment, | ||||||
4001 | VAArgOverflowSize); | ||||||
4002 | if (MS.TrackOrigins) { | ||||||
4003 | SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy, | ||||||
4004 | AMD64FpEndOffset); | ||||||
4005 | IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment, | ||||||
4006 | VAArgOverflowSize); | ||||||
4007 | } | ||||||
4008 | } | ||||||
4009 | } | ||||||
4010 | }; | ||||||
4011 | |||||||
4012 | /// MIPS64-specific implementation of VarArgHelper. | ||||||
4013 | struct VarArgMIPS64Helper : public VarArgHelper { | ||||||
4014 | Function &F; | ||||||
4015 | MemorySanitizer &MS; | ||||||
4016 | MemorySanitizerVisitor &MSV; | ||||||
4017 | Value *VAArgTLSCopy = nullptr; | ||||||
4018 | Value *VAArgSize = nullptr; | ||||||
4019 | |||||||
4020 | SmallVector<CallInst*, 16> VAStartInstrumentationList; | ||||||
4021 | |||||||
4022 | VarArgMIPS64Helper(Function &F, MemorySanitizer &MS, | ||||||
4023 | MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {} | ||||||
4024 | |||||||
4025 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { | ||||||
4026 | unsigned VAArgOffset = 0; | ||||||
4027 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
4028 | for (CallSite::arg_iterator ArgIt = CS.arg_begin() + | ||||||
4029 | CS.getFunctionType()->getNumParams(), End = CS.arg_end(); | ||||||
4030 | ArgIt != End; ++ArgIt) { | ||||||
4031 | Triple TargetTriple(F.getParent()->getTargetTriple()); | ||||||
4032 | Value *A = *ArgIt; | ||||||
4033 | Value *Base; | ||||||
4034 | uint64_t ArgSize = DL.getTypeAllocSize(A->getType()); | ||||||
4035 | if (TargetTriple.getArch() == Triple::mips64) { | ||||||
4036 | // Adjusting the shadow for argument with size < 8 to match the placement | ||||||
4037 | // of bits in big endian system | ||||||
4038 | if (ArgSize < 8) | ||||||
4039 | VAArgOffset += (8 - ArgSize); | ||||||
4040 | } | ||||||
4041 | Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize); | ||||||
4042 | VAArgOffset += ArgSize; | ||||||
4043 | VAArgOffset = alignTo(VAArgOffset, 8); | ||||||
4044 | if (!Base) | ||||||
4045 | continue; | ||||||
4046 | IRB.CreateAlignedStore(MSV.getShadow(A), Base, | ||||||
4047 | kShadowTLSAlignment.value()); | ||||||
4048 | } | ||||||
4049 | |||||||
4050 | Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset); | ||||||
4051 | // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of | ||||||
4052 | // a new class member i.e. it is the total size of all VarArgs. | ||||||
4053 | IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS); | ||||||
4054 | } | ||||||
4055 | |||||||
4056 | /// Compute the shadow address for a given va_arg. | ||||||
4057 | Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, | ||||||
4058 | unsigned ArgOffset, unsigned ArgSize) { | ||||||
4059 | // Make sure we don't overflow __msan_va_arg_tls. | ||||||
4060 | if (ArgOffset + ArgSize > kParamTLSSize) | ||||||
4061 | return nullptr; | ||||||
4062 | Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); | ||||||
4063 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
4064 | return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), | ||||||
4065 | "_msarg"); | ||||||
4066 | } | ||||||
4067 | |||||||
4068 | void visitVAStartInst(VAStartInst &I) override { | ||||||
4069 | IRBuilder<> IRB(&I); | ||||||
4070 | VAStartInstrumentationList.push_back(&I); | ||||||
4071 | Value *VAListTag = I.getArgOperand(0); | ||||||
4072 | Value *ShadowPtr, *OriginPtr; | ||||||
4073 | const Align Alignment = Align(8); | ||||||
4074 | std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( | ||||||
4075 | VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); | ||||||
4076 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
4077 | /* size */ 8, Alignment, false); | ||||||
4078 | } | ||||||
4079 | |||||||
4080 | void visitVACopyInst(VACopyInst &I) override { | ||||||
4081 | IRBuilder<> IRB(&I); | ||||||
4082 | VAStartInstrumentationList.push_back(&I); | ||||||
4083 | Value *VAListTag = I.getArgOperand(0); | ||||||
4084 | Value *ShadowPtr, *OriginPtr; | ||||||
4085 | const Align Alignment = Align(8); | ||||||
4086 | std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( | ||||||
4087 | VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); | ||||||
4088 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
4089 | /* size */ 8, Alignment, false); | ||||||
4090 | } | ||||||
4091 | |||||||
4092 | void finalizeInstrumentation() override { | ||||||
4093 | assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4094, __PRETTY_FUNCTION__)) | ||||||
4094 | "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4094, __PRETTY_FUNCTION__)); | ||||||
4095 | IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); | ||||||
4096 | VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); | ||||||
4097 | Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), | ||||||
4098 | VAArgSize); | ||||||
4099 | |||||||
4100 | if (!VAStartInstrumentationList.empty()) { | ||||||
4101 | // If there is a va_start in this function, make a backup copy of | ||||||
4102 | // va_arg_tls somewhere in the function entry block. | ||||||
4103 | VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); | ||||||
4104 | IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); | ||||||
4105 | } | ||||||
4106 | |||||||
4107 | // Instrument va_start. | ||||||
4108 | // Copy va_list shadow from the backup copy of the TLS contents. | ||||||
4109 | for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) { | ||||||
4110 | CallInst *OrigInst = VAStartInstrumentationList[i]; | ||||||
4111 | IRBuilder<> IRB(OrigInst->getNextNode()); | ||||||
4112 | Value *VAListTag = OrigInst->getArgOperand(0); | ||||||
4113 | Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); | ||||||
4114 | Value *RegSaveAreaPtrPtr = | ||||||
4115 | IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | ||||||
4116 | PointerType::get(RegSaveAreaPtrTy, 0)); | ||||||
4117 | Value *RegSaveAreaPtr = | ||||||
4118 | IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); | ||||||
4119 | Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; | ||||||
4120 | const Align Alignment = Align(8); | ||||||
4121 | std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = | ||||||
4122 | MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
4123 | Alignment, /*isStore*/ true); | ||||||
4124 | IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, | ||||||
4125 | CopySize); | ||||||
4126 | } | ||||||
4127 | } | ||||||
4128 | }; | ||||||
4129 | |||||||
4130 | /// AArch64-specific implementation of VarArgHelper. | ||||||
4131 | struct VarArgAArch64Helper : public VarArgHelper { | ||||||
4132 | static const unsigned kAArch64GrArgSize = 64; | ||||||
4133 | static const unsigned kAArch64VrArgSize = 128; | ||||||
4134 | |||||||
4135 | static const unsigned AArch64GrBegOffset = 0; | ||||||
4136 | static const unsigned AArch64GrEndOffset = kAArch64GrArgSize; | ||||||
4137 | // Make VR space aligned to 16 bytes. | ||||||
4138 | static const unsigned AArch64VrBegOffset = AArch64GrEndOffset; | ||||||
4139 | static const unsigned AArch64VrEndOffset = AArch64VrBegOffset | ||||||
4140 | + kAArch64VrArgSize; | ||||||
4141 | static const unsigned AArch64VAEndOffset = AArch64VrEndOffset; | ||||||
4142 | |||||||
4143 | Function &F; | ||||||
4144 | MemorySanitizer &MS; | ||||||
4145 | MemorySanitizerVisitor &MSV; | ||||||
4146 | Value *VAArgTLSCopy = nullptr; | ||||||
4147 | Value *VAArgOverflowSize = nullptr; | ||||||
4148 | |||||||
4149 | SmallVector<CallInst*, 16> VAStartInstrumentationList; | ||||||
4150 | |||||||
4151 | enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory }; | ||||||
4152 | |||||||
4153 | VarArgAArch64Helper(Function &F, MemorySanitizer &MS, | ||||||
4154 | MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {} | ||||||
4155 | |||||||
4156 | ArgKind classifyArgument(Value* arg) { | ||||||
4157 | Type *T = arg->getType(); | ||||||
4158 | if (T->isFPOrFPVectorTy()) | ||||||
4159 | return AK_FloatingPoint; | ||||||
4160 | if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64) | ||||||
4161 | || (T->isPointerTy())) | ||||||
4162 | return AK_GeneralPurpose; | ||||||
4163 | return AK_Memory; | ||||||
4164 | } | ||||||
4165 | |||||||
4166 | // The instrumentation stores the argument shadow in a non ABI-specific | ||||||
4167 | // format because it does not know which argument is named (since Clang, | ||||||
4168 | // like x86_64 case, lowers the va_args in the frontend and this pass only | ||||||
4169 | // sees the low level code that deals with va_list internals). | ||||||
4170 | // The first seven GR registers are saved in the first 56 bytes of the | ||||||
4171 | // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then | ||||||
4172 | // the remaining arguments. | ||||||
4173 | // Using constant offset within the va_arg TLS array allows fast copy | ||||||
4174 | // in the finalize instrumentation. | ||||||
4175 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { | ||||||
4176 | unsigned GrOffset = AArch64GrBegOffset; | ||||||
4177 | unsigned VrOffset = AArch64VrBegOffset; | ||||||
4178 | unsigned OverflowOffset = AArch64VAEndOffset; | ||||||
4179 | |||||||
4180 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
4181 | for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); | ||||||
4182 | ArgIt != End; ++ArgIt) { | ||||||
4183 | Value *A = *ArgIt; | ||||||
4184 | unsigned ArgNo = CS.getArgumentNo(ArgIt); | ||||||
4185 | bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams(); | ||||||
4186 | ArgKind AK = classifyArgument(A); | ||||||
4187 | if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset) | ||||||
4188 | AK = AK_Memory; | ||||||
4189 | if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset) | ||||||
4190 | AK = AK_Memory; | ||||||
4191 | Value *Base; | ||||||
4192 | switch (AK) { | ||||||
4193 | case AK_GeneralPurpose: | ||||||
4194 | Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8); | ||||||
4195 | GrOffset += 8; | ||||||
4196 | break; | ||||||
4197 | case AK_FloatingPoint: | ||||||
4198 | Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8); | ||||||
4199 | VrOffset += 16; | ||||||
4200 | break; | ||||||
4201 | case AK_Memory: | ||||||
4202 | // Don't count fixed arguments in the overflow area - va_start will | ||||||
4203 | // skip right over them. | ||||||
4204 | if (IsFixed) | ||||||
4205 | continue; | ||||||
4206 | uint64_t ArgSize = DL.getTypeAllocSize(A->getType()); | ||||||
4207 | Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, | ||||||
4208 | alignTo(ArgSize, 8)); | ||||||
4209 | OverflowOffset += alignTo(ArgSize, 8); | ||||||
4210 | break; | ||||||
4211 | } | ||||||
4212 | // Count Gp/Vr fixed arguments to their respective offsets, but don't | ||||||
4213 | // bother to actually store a shadow. | ||||||
4214 | if (IsFixed) | ||||||
4215 | continue; | ||||||
4216 | if (!Base) | ||||||
4217 | continue; | ||||||
4218 | IRB.CreateAlignedStore(MSV.getShadow(A), Base, | ||||||
4219 | kShadowTLSAlignment.value()); | ||||||
4220 | } | ||||||
4221 | Constant *OverflowSize = | ||||||
4222 | ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset); | ||||||
4223 | IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS); | ||||||
4224 | } | ||||||
4225 | |||||||
4226 | /// Compute the shadow address for a given va_arg. | ||||||
4227 | Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, | ||||||
4228 | unsigned ArgOffset, unsigned ArgSize) { | ||||||
4229 | // Make sure we don't overflow __msan_va_arg_tls. | ||||||
4230 | if (ArgOffset + ArgSize > kParamTLSSize) | ||||||
4231 | return nullptr; | ||||||
4232 | Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); | ||||||
4233 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
4234 | return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), | ||||||
4235 | "_msarg"); | ||||||
4236 | } | ||||||
4237 | |||||||
4238 | void visitVAStartInst(VAStartInst &I) override { | ||||||
4239 | IRBuilder<> IRB(&I); | ||||||
4240 | VAStartInstrumentationList.push_back(&I); | ||||||
4241 | Value *VAListTag = I.getArgOperand(0); | ||||||
4242 | Value *ShadowPtr, *OriginPtr; | ||||||
4243 | const Align Alignment = Align(8); | ||||||
4244 | std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( | ||||||
4245 | VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); | ||||||
4246 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
4247 | /* size */ 32, Alignment, false); | ||||||
4248 | } | ||||||
4249 | |||||||
4250 | void visitVACopyInst(VACopyInst &I) override { | ||||||
4251 | IRBuilder<> IRB(&I); | ||||||
4252 | VAStartInstrumentationList.push_back(&I); | ||||||
4253 | Value *VAListTag = I.getArgOperand(0); | ||||||
4254 | Value *ShadowPtr, *OriginPtr; | ||||||
4255 | const Align Alignment = Align(8); | ||||||
4256 | std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( | ||||||
4257 | VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); | ||||||
4258 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
4259 | /* size */ 32, Alignment, false); | ||||||
4260 | } | ||||||
4261 | |||||||
4262 | // Retrieve a va_list field of 'void*' size. | ||||||
4263 | Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) { | ||||||
4264 | Value *SaveAreaPtrPtr = | ||||||
4265 | IRB.CreateIntToPtr( | ||||||
4266 | IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | ||||||
4267 | ConstantInt::get(MS.IntptrTy, offset)), | ||||||
4268 | Type::getInt64PtrTy(*MS.C)); | ||||||
4269 | return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr); | ||||||
4270 | } | ||||||
4271 | |||||||
4272 | // Retrieve a va_list field of 'int' size. | ||||||
4273 | Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) { | ||||||
4274 | Value *SaveAreaPtr = | ||||||
4275 | IRB.CreateIntToPtr( | ||||||
4276 | IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | ||||||
4277 | ConstantInt::get(MS.IntptrTy, offset)), | ||||||
4278 | Type::getInt32PtrTy(*MS.C)); | ||||||
4279 | Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr); | ||||||
4280 | return IRB.CreateSExt(SaveArea32, MS.IntptrTy); | ||||||
4281 | } | ||||||
4282 | |||||||
4283 | void finalizeInstrumentation() override { | ||||||
4284 | assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4285, __PRETTY_FUNCTION__)) | ||||||
4285 | "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4285, __PRETTY_FUNCTION__)); | ||||||
4286 | if (!VAStartInstrumentationList.empty()) { | ||||||
4287 | // If there is a va_start in this function, make a backup copy of | ||||||
4288 | // va_arg_tls somewhere in the function entry block. | ||||||
4289 | IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); | ||||||
4290 | VAArgOverflowSize = | ||||||
4291 | IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); | ||||||
4292 | Value *CopySize = | ||||||
4293 | IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), | ||||||
4294 | VAArgOverflowSize); | ||||||
4295 | VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); | ||||||
4296 | IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); | ||||||
4297 | } | ||||||
4298 | |||||||
4299 | Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize); | ||||||
4300 | Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize); | ||||||
4301 | |||||||
4302 | // Instrument va_start, copy va_list shadow from the backup copy of | ||||||
4303 | // the TLS contents. | ||||||
4304 | for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) { | ||||||
4305 | CallInst *OrigInst = VAStartInstrumentationList[i]; | ||||||
4306 | IRBuilder<> IRB(OrigInst->getNextNode()); | ||||||
4307 | |||||||
4308 | Value *VAListTag = OrigInst->getArgOperand(0); | ||||||
4309 | |||||||
4310 | // The variadic ABI for AArch64 creates two areas to save the incoming | ||||||
4311 | // argument registers (one for 64-bit general register xn-x7 and another | ||||||
4312 | // for 128-bit FP/SIMD vn-v7). | ||||||
4313 | // We need then to propagate the shadow arguments on both regions | ||||||
4314 | // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'. | ||||||
4315 | // The remaning arguments are saved on shadow for 'va::stack'. | ||||||
4316 | // One caveat is it requires only to propagate the non-named arguments, | ||||||
4317 | // however on the call site instrumentation 'all' the arguments are | ||||||
4318 | // saved. So to copy the shadow values from the va_arg TLS array | ||||||
4319 | // we need to adjust the offset for both GR and VR fields based on | ||||||
4320 | // the __{gr,vr}_offs value (since they are stores based on incoming | ||||||
4321 | // named arguments). | ||||||
4322 | |||||||
4323 | // Read the stack pointer from the va_list. | ||||||
4324 | Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0); | ||||||
4325 | |||||||
4326 | // Read both the __gr_top and __gr_off and add them up. | ||||||
4327 | Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8); | ||||||
4328 | Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24); | ||||||
4329 | |||||||
4330 | Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea); | ||||||
4331 | |||||||
4332 | // Read both the __vr_top and __vr_off and add them up. | ||||||
4333 | Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16); | ||||||
4334 | Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28); | ||||||
4335 | |||||||
4336 | Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea); | ||||||
4337 | |||||||
4338 | // It does not know how many named arguments is being used and, on the | ||||||
4339 | // callsite all the arguments were saved. Since __gr_off is defined as | ||||||
4340 | // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic | ||||||
4341 | // argument by ignoring the bytes of shadow from named arguments. | ||||||
4342 | Value *GrRegSaveAreaShadowPtrOff = | ||||||
4343 | IRB.CreateAdd(GrArgSize, GrOffSaveArea); | ||||||
4344 | |||||||
4345 | Value *GrRegSaveAreaShadowPtr = | ||||||
4346 | MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
4347 | Align(8), /*isStore*/ true) | ||||||
4348 | .first; | ||||||
4349 | |||||||
4350 | Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy, | ||||||
4351 | GrRegSaveAreaShadowPtrOff); | ||||||
4352 | Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff); | ||||||
4353 | |||||||
4354 | IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8), | ||||||
4355 | GrCopySize); | ||||||
4356 | |||||||
4357 | // Again, but for FP/SIMD values. | ||||||
4358 | Value *VrRegSaveAreaShadowPtrOff = | ||||||
4359 | IRB.CreateAdd(VrArgSize, VrOffSaveArea); | ||||||
4360 | |||||||
4361 | Value *VrRegSaveAreaShadowPtr = | ||||||
4362 | MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
4363 | Align(8), /*isStore*/ true) | ||||||
4364 | .first; | ||||||
4365 | |||||||
4366 | Value *VrSrcPtr = IRB.CreateInBoundsGEP( | ||||||
4367 | IRB.getInt8Ty(), | ||||||
4368 | IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy, | ||||||
4369 | IRB.getInt32(AArch64VrBegOffset)), | ||||||
4370 | VrRegSaveAreaShadowPtrOff); | ||||||
4371 | Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff); | ||||||
4372 | |||||||
4373 | IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8), | ||||||
4374 | VrCopySize); | ||||||
4375 | |||||||
4376 | // And finally for remaining arguments. | ||||||
4377 | Value *StackSaveAreaShadowPtr = | ||||||
4378 | MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
4379 | Align(16), /*isStore*/ true) | ||||||
4380 | .first; | ||||||
4381 | |||||||
4382 | Value *StackSrcPtr = | ||||||
4383 | IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy, | ||||||
4384 | IRB.getInt32(AArch64VAEndOffset)); | ||||||
4385 | |||||||
4386 | IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr, | ||||||
4387 | Align(16), VAArgOverflowSize); | ||||||
4388 | } | ||||||
4389 | } | ||||||
4390 | }; | ||||||
4391 | |||||||
4392 | /// PowerPC64-specific implementation of VarArgHelper. | ||||||
4393 | struct VarArgPowerPC64Helper : public VarArgHelper { | ||||||
4394 | Function &F; | ||||||
4395 | MemorySanitizer &MS; | ||||||
4396 | MemorySanitizerVisitor &MSV; | ||||||
4397 | Value *VAArgTLSCopy = nullptr; | ||||||
4398 | Value *VAArgSize = nullptr; | ||||||
4399 | |||||||
4400 | SmallVector<CallInst*, 16> VAStartInstrumentationList; | ||||||
4401 | |||||||
4402 | VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS, | ||||||
4403 | MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {} | ||||||
4404 | |||||||
4405 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override { | ||||||
4406 | // For PowerPC, we need to deal with alignment of stack arguments - | ||||||
4407 | // they are mostly aligned to 8 bytes, but vectors and i128 arrays | ||||||
4408 | // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes, | ||||||
4409 | // and QPX vectors are aligned to 32 bytes. For that reason, we | ||||||
4410 | // compute current offset from stack pointer (which is always properly | ||||||
4411 | // aligned), and offset for the first vararg, then subtract them. | ||||||
4412 | unsigned VAArgBase; | ||||||
4413 | Triple TargetTriple(F.getParent()->getTargetTriple()); | ||||||
4414 | // Parameter save area starts at 48 bytes from frame pointer for ABIv1, | ||||||
4415 | // and 32 bytes for ABIv2. This is usually determined by target | ||||||
4416 | // endianness, but in theory could be overriden by function attribute. | ||||||
4417 | // For simplicity, we ignore it here (it'd only matter for QPX vectors). | ||||||
4418 | if (TargetTriple.getArch() == Triple::ppc64) | ||||||
4419 | VAArgBase = 48; | ||||||
4420 | else | ||||||
4421 | VAArgBase = 32; | ||||||
4422 | unsigned VAArgOffset = VAArgBase; | ||||||
4423 | const DataLayout &DL = F.getParent()->getDataLayout(); | ||||||
4424 | for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end(); | ||||||
4425 | ArgIt != End; ++ArgIt) { | ||||||
4426 | Value *A = *ArgIt; | ||||||
4427 | unsigned ArgNo = CS.getArgumentNo(ArgIt); | ||||||
4428 | bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams(); | ||||||
4429 | bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal); | ||||||
4430 | if (IsByVal) { | ||||||
4431 | assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4431, __PRETTY_FUNCTION__)); | ||||||
4432 | Type *RealTy = A->getType()->getPointerElementType(); | ||||||
4433 | uint64_t ArgSize = DL.getTypeAllocSize(RealTy); | ||||||
4434 | uint64_t ArgAlign = CS.getParamAlignment(ArgNo); | ||||||
4435 | if (ArgAlign < 8) | ||||||
4436 | ArgAlign = 8; | ||||||
4437 | VAArgOffset = alignTo(VAArgOffset, ArgAlign); | ||||||
4438 | if (!IsFixed) { | ||||||
4439 | Value *Base = getShadowPtrForVAArgument( | ||||||
4440 | RealTy, IRB, VAArgOffset - VAArgBase, ArgSize); | ||||||
4441 | if (Base) { | ||||||
4442 | Value *AShadowPtr, *AOriginPtr; | ||||||
4443 | std::tie(AShadowPtr, AOriginPtr) = | ||||||
4444 | MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), | ||||||
4445 | kShadowTLSAlignment, /*isStore*/ false); | ||||||
4446 | |||||||
4447 | IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr, | ||||||
4448 | kShadowTLSAlignment, ArgSize); | ||||||
4449 | } | ||||||
4450 | } | ||||||
4451 | VAArgOffset += alignTo(ArgSize, 8); | ||||||
4452 | } else { | ||||||
4453 | Value *Base; | ||||||
4454 | uint64_t ArgSize = DL.getTypeAllocSize(A->getType()); | ||||||
4455 | uint64_t ArgAlign = 8; | ||||||
4456 | if (A->getType()->isArrayTy()) { | ||||||
4457 | // Arrays are aligned to element size, except for long double | ||||||
4458 | // arrays, which are aligned to 8 bytes. | ||||||
4459 | Type *ElementTy = A->getType()->getArrayElementType(); | ||||||
4460 | if (!ElementTy->isPPC_FP128Ty()) | ||||||
4461 | ArgAlign = DL.getTypeAllocSize(ElementTy); | ||||||
4462 | } else if (A->getType()->isVectorTy()) { | ||||||
4463 | // Vectors are naturally aligned. | ||||||
4464 | ArgAlign = DL.getTypeAllocSize(A->getType()); | ||||||
4465 | } | ||||||
4466 | if (ArgAlign < 8) | ||||||
4467 | ArgAlign = 8; | ||||||
4468 | VAArgOffset = alignTo(VAArgOffset, ArgAlign); | ||||||
4469 | if (DL.isBigEndian()) { | ||||||
4470 | // Adjusting the shadow for argument with size < 8 to match the placement | ||||||
4471 | // of bits in big endian system | ||||||
4472 | if (ArgSize < 8) | ||||||
4473 | VAArgOffset += (8 - ArgSize); | ||||||
4474 | } | ||||||
4475 | if (!IsFixed) { | ||||||
4476 | Base = getShadowPtrForVAArgument(A->getType(), IRB, | ||||||
4477 | VAArgOffset - VAArgBase, ArgSize); | ||||||
4478 | if (Base) | ||||||
4479 | IRB.CreateAlignedStore(MSV.getShadow(A), Base, | ||||||
4480 | kShadowTLSAlignment.value()); | ||||||
4481 | } | ||||||
4482 | VAArgOffset += ArgSize; | ||||||
4483 | VAArgOffset = alignTo(VAArgOffset, 8); | ||||||
4484 | } | ||||||
4485 | if (IsFixed) | ||||||
4486 | VAArgBase = VAArgOffset; | ||||||
4487 | } | ||||||
4488 | |||||||
4489 | Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), | ||||||
4490 | VAArgOffset - VAArgBase); | ||||||
4491 | // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of | ||||||
4492 | // a new class member i.e. it is the total size of all VarArgs. | ||||||
4493 | IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS); | ||||||
4494 | } | ||||||
4495 | |||||||
4496 | /// Compute the shadow address for a given va_arg. | ||||||
4497 | Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, | ||||||
4498 | unsigned ArgOffset, unsigned ArgSize) { | ||||||
4499 | // Make sure we don't overflow __msan_va_arg_tls. | ||||||
4500 | if (ArgOffset + ArgSize > kParamTLSSize) | ||||||
4501 | return nullptr; | ||||||
4502 | Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy); | ||||||
4503 | Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset)); | ||||||
4504 | return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0), | ||||||
4505 | "_msarg"); | ||||||
4506 | } | ||||||
4507 | |||||||
4508 | void visitVAStartInst(VAStartInst &I) override { | ||||||
4509 | IRBuilder<> IRB(&I); | ||||||
4510 | VAStartInstrumentationList.push_back(&I); | ||||||
4511 | Value *VAListTag = I.getArgOperand(0); | ||||||
4512 | Value *ShadowPtr, *OriginPtr; | ||||||
4513 | const Align Alignment = Align(8); | ||||||
4514 | std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( | ||||||
4515 | VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); | ||||||
4516 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
4517 | /* size */ 8, Alignment, false); | ||||||
4518 | } | ||||||
4519 | |||||||
4520 | void visitVACopyInst(VACopyInst &I) override { | ||||||
4521 | IRBuilder<> IRB(&I); | ||||||
4522 | Value *VAListTag = I.getArgOperand(0); | ||||||
4523 | Value *ShadowPtr, *OriginPtr; | ||||||
4524 | const Align Alignment = Align(8); | ||||||
4525 | std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr( | ||||||
4526 | VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true); | ||||||
4527 | // Unpoison the whole __va_list_tag. | ||||||
4528 | // FIXME: magic ABI constants. | ||||||
4529 | IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()), | ||||||
4530 | /* size */ 8, Alignment, false); | ||||||
4531 | } | ||||||
4532 | |||||||
4533 | void finalizeInstrumentation() override { | ||||||
4534 | assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4535, __PRETTY_FUNCTION__)) | ||||||
4535 | "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice" ) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp" , 4535, __PRETTY_FUNCTION__)); | ||||||
4536 | IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI()); | ||||||
4537 | VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS); | ||||||
4538 | Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), | ||||||
4539 | VAArgSize); | ||||||
4540 | |||||||
4541 | if (!VAStartInstrumentationList.empty()) { | ||||||
4542 | // If there is a va_start in this function, make a backup copy of | ||||||
4543 | // va_arg_tls somewhere in the function entry block. | ||||||
4544 | VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize); | ||||||
4545 | IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize); | ||||||
4546 | } | ||||||
4547 | |||||||
4548 | // Instrument va_start. | ||||||
4549 | // Copy va_list shadow from the backup copy of the TLS contents. | ||||||
4550 | for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) { | ||||||
4551 | CallInst *OrigInst = VAStartInstrumentationList[i]; | ||||||
4552 | IRBuilder<> IRB(OrigInst->getNextNode()); | ||||||
4553 | Value *VAListTag = OrigInst->getArgOperand(0); | ||||||
4554 | Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C); | ||||||
4555 | Value *RegSaveAreaPtrPtr = | ||||||
4556 | IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy), | ||||||
4557 | PointerType::get(RegSaveAreaPtrTy, 0)); | ||||||
4558 | Value *RegSaveAreaPtr = | ||||||
4559 | IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr); | ||||||
4560 | Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr; | ||||||
4561 | const Align Alignment = Align(8); | ||||||
4562 | std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) = | ||||||
4563 | MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), | ||||||
4564 | Alignment, /*isStore*/ true); | ||||||
4565 | IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment, | ||||||
4566 | CopySize); | ||||||
4567 | } | ||||||
4568 | } | ||||||
4569 | }; | ||||||
4570 | |||||||
4571 | /// A no-op implementation of VarArgHelper. | ||||||
4572 | struct VarArgNoOpHelper : public VarArgHelper { | ||||||
4573 | VarArgNoOpHelper(Function &F, MemorySanitizer &MS, | ||||||
4574 | MemorySanitizerVisitor &MSV) {} | ||||||
4575 | |||||||
4576 | void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {} | ||||||
4577 | |||||||
4578 | void visitVAStartInst(VAStartInst &I) override {} | ||||||
4579 | |||||||
4580 | void visitVACopyInst(VACopyInst &I) override {} | ||||||
4581 | |||||||
4582 | void finalizeInstrumentation() override {} | ||||||
4583 | }; | ||||||
4584 | |||||||
4585 | } // end anonymous namespace | ||||||
4586 | |||||||
4587 | static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, | ||||||
4588 | MemorySanitizerVisitor &Visitor) { | ||||||
4589 | // VarArg handling is only implemented on AMD64. False positives are possible | ||||||
4590 | // on other platforms. | ||||||
4591 | Triple TargetTriple(Func.getParent()->getTargetTriple()); | ||||||
4592 | if (TargetTriple.getArch() == Triple::x86_64) | ||||||
4593 | return new VarArgAMD64Helper(Func, Msan, Visitor); | ||||||
4594 | else if (TargetTriple.isMIPS64()) | ||||||
4595 | return new VarArgMIPS64Helper(Func, Msan, Visitor); | ||||||
4596 | else if (TargetTriple.getArch() == Triple::aarch64) | ||||||
4597 | return new VarArgAArch64Helper(Func, Msan, Visitor); | ||||||
4598 | else if (TargetTriple.getArch() == Triple::ppc64 || | ||||||
4599 | TargetTriple.getArch() == Triple::ppc64le) | ||||||
4600 | return new VarArgPowerPC64Helper(Func, Msan, Visitor); | ||||||
4601 | else | ||||||
4602 | return new VarArgNoOpHelper(Func, Msan, Visitor); | ||||||
4603 | } | ||||||
4604 | |||||||
4605 | bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) { | ||||||
4606 | if (!CompileKernel && F.getName() == kMsanModuleCtorName) | ||||||
4607 | return false; | ||||||
4608 | |||||||
4609 | MemorySanitizerVisitor Visitor(F, *this, TLI); | ||||||
4610 | |||||||
4611 | // Clear out readonly/readnone attributes. | ||||||
4612 | AttrBuilder B; | ||||||
4613 | B.addAttribute(Attribute::ReadOnly) | ||||||
4614 | .addAttribute(Attribute::ReadNone) | ||||||
4615 | .addAttribute(Attribute::WriteOnly) | ||||||
4616 | .addAttribute(Attribute::ArgMemOnly) | ||||||
4617 | .addAttribute(Attribute::Speculatable); | ||||||
4618 | F.removeAttributes(AttributeList::FunctionIndex, B); | ||||||
4619 | |||||||
4620 | return Visitor.runOnFunction(); | ||||||
4621 | } |
1 | //===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the declaration of the Type class. For more "Type" |
10 | // stuff, look in DerivedTypes.h. |
11 | // |
12 | //===----------------------------------------------------------------------===// |
13 | |
14 | #ifndef LLVM_IR_TYPE_H |
15 | #define LLVM_IR_TYPE_H |
16 | |
17 | #include "llvm/ADT/APFloat.h" |
18 | #include "llvm/ADT/ArrayRef.h" |
19 | #include "llvm/ADT/SmallPtrSet.h" |
20 | #include "llvm/Support/CBindingWrapping.h" |
21 | #include "llvm/Support/Casting.h" |
22 | #include "llvm/Support/Compiler.h" |
23 | #include "llvm/Support/ErrorHandling.h" |
24 | #include "llvm/Support/TypeSize.h" |
25 | #include <cassert> |
26 | #include <cstdint> |
27 | #include <iterator> |
28 | |
29 | namespace llvm { |
30 | |
31 | template<class GraphType> struct GraphTraits; |
32 | class IntegerType; |
33 | class LLVMContext; |
34 | class PointerType; |
35 | class raw_ostream; |
36 | class StringRef; |
37 | |
38 | /// The instances of the Type class are immutable: once they are created, |
39 | /// they are never changed. Also note that only one instance of a particular |
40 | /// type is ever created. Thus seeing if two types are equal is a matter of |
41 | /// doing a trivial pointer comparison. To enforce that no two equal instances |
42 | /// are created, Type instances can only be created via static factory methods |
43 | /// in class Type and in derived classes. Once allocated, Types are never |
44 | /// free'd. |
45 | /// |
46 | class Type { |
47 | public: |
48 | //===--------------------------------------------------------------------===// |
49 | /// Definitions of all of the base types for the Type system. Based on this |
50 | /// value, you can cast to a class defined in DerivedTypes.h. |
51 | /// Note: If you add an element to this, you need to add an element to the |
52 | /// Type::getPrimitiveType function, or else things will break! |
53 | /// Also update LLVMTypeKind and LLVMGetTypeKind () in the C binding. |
54 | /// |
55 | enum TypeID { |
56 | // PrimitiveTypes - make sure LastPrimitiveTyID stays up to date. |
57 | VoidTyID = 0, ///< 0: type with no size |
58 | HalfTyID, ///< 1: 16-bit floating point type |
59 | FloatTyID, ///< 2: 32-bit floating point type |
60 | DoubleTyID, ///< 3: 64-bit floating point type |
61 | X86_FP80TyID, ///< 4: 80-bit floating point type (X87) |
62 | FP128TyID, ///< 5: 128-bit floating point type (112-bit mantissa) |
63 | PPC_FP128TyID, ///< 6: 128-bit floating point type (two 64-bits, PowerPC) |
64 | LabelTyID, ///< 7: Labels |
65 | MetadataTyID, ///< 8: Metadata |
66 | X86_MMXTyID, ///< 9: MMX vectors (64 bits, X86 specific) |
67 | TokenTyID, ///< 10: Tokens |
68 | |
69 | // Derived types... see DerivedTypes.h file. |
70 | // Make sure FirstDerivedTyID stays up to date! |
71 | IntegerTyID, ///< 11: Arbitrary bit width integers |
72 | FunctionTyID, ///< 12: Functions |
73 | StructTyID, ///< 13: Structures |
74 | ArrayTyID, ///< 14: Arrays |
75 | PointerTyID, ///< 15: Pointers |
76 | VectorTyID ///< 16: SIMD 'packed' format, or other vector type |
77 | }; |
78 | |
79 | private: |
80 | /// This refers to the LLVMContext in which this type was uniqued. |
81 | LLVMContext &Context; |
82 | |
83 | TypeID ID : 8; // The current base type of this type. |
84 | unsigned SubclassData : 24; // Space for subclasses to store data. |
85 | // Note that this should be synchronized with |
86 | // MAX_INT_BITS value in IntegerType class. |
87 | |
88 | protected: |
89 | friend class LLVMContextImpl; |
90 | |
91 | explicit Type(LLVMContext &C, TypeID tid) |
92 | : Context(C), ID(tid), SubclassData(0) {} |
93 | ~Type() = default; |
94 | |
95 | unsigned getSubclassData() const { return SubclassData; } |
96 | |
97 | void setSubclassData(unsigned val) { |
98 | SubclassData = val; |
99 | // Ensure we don't have any accidental truncation. |
100 | assert(getSubclassData() == val && "Subclass data too large for field")((getSubclassData() == val && "Subclass data too large for field" ) ? static_cast<void> (0) : __assert_fail ("getSubclassData() == val && \"Subclass data too large for field\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 100, __PRETTY_FUNCTION__)); |
101 | } |
102 | |
103 | /// Keeps track of how many Type*'s there are in the ContainedTys list. |
104 | unsigned NumContainedTys = 0; |
105 | |
106 | /// A pointer to the array of Types contained by this Type. For example, this |
107 | /// includes the arguments of a function type, the elements of a structure, |
108 | /// the pointee of a pointer, the element type of an array, etc. This pointer |
109 | /// may be 0 for types that don't contain other types (Integer, Double, |
110 | /// Float). |
111 | Type * const *ContainedTys = nullptr; |
112 | |
113 | static bool isSequentialType(TypeID TyID) { |
114 | return TyID == ArrayTyID || TyID == VectorTyID; |
115 | } |
116 | |
117 | public: |
118 | /// Print the current type. |
119 | /// Omit the type details if \p NoDetails == true. |
120 | /// E.g., let %st = type { i32, i16 } |
121 | /// When \p NoDetails is true, we only print %st. |
122 | /// Put differently, \p NoDetails prints the type as if |
123 | /// inlined with the operands when printing an instruction. |
124 | void print(raw_ostream &O, bool IsForDebug = false, |
125 | bool NoDetails = false) const; |
126 | |
127 | void dump() const; |
128 | |
129 | /// Return the LLVMContext in which this type was uniqued. |
130 | LLVMContext &getContext() const { return Context; } |
131 | |
132 | //===--------------------------------------------------------------------===// |
133 | // Accessors for working with types. |
134 | // |
135 | |
136 | /// Return the type id for the type. This will return one of the TypeID enum |
137 | /// elements defined above. |
138 | TypeID getTypeID() const { return ID; } |
139 | |
140 | /// Return true if this is 'void'. |
141 | bool isVoidTy() const { return getTypeID() == VoidTyID; } |
142 | |
143 | /// Return true if this is 'half', a 16-bit IEEE fp type. |
144 | bool isHalfTy() const { return getTypeID() == HalfTyID; } |
145 | |
146 | /// Return true if this is 'float', a 32-bit IEEE fp type. |
147 | bool isFloatTy() const { return getTypeID() == FloatTyID; } |
148 | |
149 | /// Return true if this is 'double', a 64-bit IEEE fp type. |
150 | bool isDoubleTy() const { return getTypeID() == DoubleTyID; } |
151 | |
152 | /// Return true if this is x86 long double. |
153 | bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; } |
154 | |
155 | /// Return true if this is 'fp128'. |
156 | bool isFP128Ty() const { return getTypeID() == FP128TyID; } |
157 | |
158 | /// Return true if this is powerpc long double. |
159 | bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; } |
160 | |
161 | /// Return true if this is one of the six floating-point types |
162 | bool isFloatingPointTy() const { |
163 | return getTypeID() == HalfTyID || getTypeID() == FloatTyID || |
164 | getTypeID() == DoubleTyID || |
165 | getTypeID() == X86_FP80TyID || getTypeID() == FP128TyID || |
166 | getTypeID() == PPC_FP128TyID; |
167 | } |
168 | |
169 | const fltSemantics &getFltSemantics() const { |
170 | switch (getTypeID()) { |
171 | case HalfTyID: return APFloat::IEEEhalf(); |
172 | case FloatTyID: return APFloat::IEEEsingle(); |
173 | case DoubleTyID: return APFloat::IEEEdouble(); |
174 | case X86_FP80TyID: return APFloat::x87DoubleExtended(); |
175 | case FP128TyID: return APFloat::IEEEquad(); |
176 | case PPC_FP128TyID: return APFloat::PPCDoubleDouble(); |
177 | default: llvm_unreachable("Invalid floating type")::llvm::llvm_unreachable_internal("Invalid floating type", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 177); |
178 | } |
179 | } |
180 | |
181 | /// Return true if this is X86 MMX. |
182 | bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; } |
183 | |
184 | /// Return true if this is a FP type or a vector of FP. |
185 | bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); } |
186 | |
187 | /// Return true if this is 'label'. |
188 | bool isLabelTy() const { return getTypeID() == LabelTyID; } |
189 | |
190 | /// Return true if this is 'metadata'. |
191 | bool isMetadataTy() const { return getTypeID() == MetadataTyID; } |
192 | |
193 | /// Return true if this is 'token'. |
194 | bool isTokenTy() const { return getTypeID() == TokenTyID; } |
195 | |
196 | /// True if this is an instance of IntegerType. |
197 | bool isIntegerTy() const { return getTypeID() == IntegerTyID; } |
198 | |
199 | /// Return true if this is an IntegerType of the given width. |
200 | bool isIntegerTy(unsigned Bitwidth) const; |
201 | |
202 | /// Return true if this is an integer type or a vector of integer types. |
203 | bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); } |
204 | |
205 | /// Return true if this is an integer type or a vector of integer types of |
206 | /// the given width. |
207 | bool isIntOrIntVectorTy(unsigned BitWidth) const { |
208 | return getScalarType()->isIntegerTy(BitWidth); |
209 | } |
210 | |
211 | /// Return true if this is an integer type or a pointer type. |
212 | bool isIntOrPtrTy() const { return isIntegerTy() || isPointerTy(); } |
213 | |
214 | /// True if this is an instance of FunctionType. |
215 | bool isFunctionTy() const { return getTypeID() == FunctionTyID; } |
216 | |
217 | /// True if this is an instance of StructType. |
218 | bool isStructTy() const { return getTypeID() == StructTyID; } |
219 | |
220 | /// True if this is an instance of ArrayType. |
221 | bool isArrayTy() const { return getTypeID() == ArrayTyID; } |
222 | |
223 | /// True if this is an instance of PointerType. |
224 | bool isPointerTy() const { return getTypeID() == PointerTyID; } |
225 | |
226 | /// Return true if this is a pointer type or a vector of pointer types. |
227 | bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); } |
228 | |
229 | /// True if this is an instance of VectorType. |
230 | bool isVectorTy() const { return getTypeID() == VectorTyID; } |
231 | |
232 | /// Return true if this type could be converted with a lossless BitCast to |
233 | /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the |
234 | /// same size only where no re-interpretation of the bits is done. |
235 | /// Determine if this type could be losslessly bitcast to Ty |
236 | bool canLosslesslyBitCastTo(Type *Ty) const; |
237 | |
238 | /// Return true if this type is empty, that is, it has no elements or all of |
239 | /// its elements are empty. |
240 | bool isEmptyTy() const; |
241 | |
242 | /// Return true if the type is "first class", meaning it is a valid type for a |
243 | /// Value. |
244 | bool isFirstClassType() const { |
245 | return getTypeID() != FunctionTyID && getTypeID() != VoidTyID; |
246 | } |
247 | |
248 | /// Return true if the type is a valid type for a register in codegen. This |
249 | /// includes all first-class types except struct and array types. |
250 | bool isSingleValueType() const { |
251 | return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() || |
252 | isPointerTy() || isVectorTy(); |
253 | } |
254 | |
255 | /// Return true if the type is an aggregate type. This means it is valid as |
256 | /// the first operand of an insertvalue or extractvalue instruction. This |
257 | /// includes struct and array types, but does not include vector types. |
258 | bool isAggregateType() const { |
259 | return getTypeID() == StructTyID || getTypeID() == ArrayTyID; |
260 | } |
261 | |
262 | /// Return true if it makes sense to take the size of this type. To get the |
263 | /// actual size for a particular target, it is reasonable to use the |
264 | /// DataLayout subsystem to do this. |
265 | bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const { |
266 | // If it's a primitive, it is always sized. |
267 | if (getTypeID() == IntegerTyID || isFloatingPointTy() || |
268 | getTypeID() == PointerTyID || |
269 | getTypeID() == X86_MMXTyID) |
270 | return true; |
271 | // If it is not something that can have a size (e.g. a function or label), |
272 | // it doesn't have a size. |
273 | if (getTypeID() != StructTyID && getTypeID() != ArrayTyID && |
274 | getTypeID() != VectorTyID) |
275 | return false; |
276 | // Otherwise we have to try harder to decide. |
277 | return isSizedDerivedType(Visited); |
278 | } |
279 | |
280 | /// Return the basic size of this type if it is a primitive type. These are |
281 | /// fixed by LLVM and are not target-dependent. |
282 | /// This will return zero if the type does not have a size or is not a |
283 | /// primitive type. |
284 | /// |
285 | /// If this is a scalable vector type, the scalable property will be set and |
286 | /// the runtime size will be a positive integer multiple of the base size. |
287 | /// |
288 | /// Note that this may not reflect the size of memory allocated for an |
289 | /// instance of the type or the number of bytes that are written when an |
290 | /// instance of the type is stored to memory. The DataLayout class provides |
291 | /// additional query functions to provide this information. |
292 | /// |
293 | TypeSize getPrimitiveSizeInBits() const LLVM_READONLY__attribute__((__pure__)); |
294 | |
295 | /// If this is a vector type, return the getPrimitiveSizeInBits value for the |
296 | /// element type. Otherwise return the getPrimitiveSizeInBits value for this |
297 | /// type. |
298 | unsigned getScalarSizeInBits() const LLVM_READONLY__attribute__((__pure__)); |
299 | |
300 | /// Return the width of the mantissa of this type. This is only valid on |
301 | /// floating-point types. If the FP type does not have a stable mantissa (e.g. |
302 | /// ppc long double), this method returns -1. |
303 | int getFPMantissaWidth() const; |
304 | |
305 | /// If this is a vector type, return the element type, otherwise return |
306 | /// 'this'. |
307 | Type *getScalarType() const { |
308 | if (isVectorTy()) |
309 | return getVectorElementType(); |
310 | return const_cast<Type*>(this); |
311 | } |
312 | |
313 | //===--------------------------------------------------------------------===// |
314 | // Type Iteration support. |
315 | // |
316 | using subtype_iterator = Type * const *; |
317 | |
318 | subtype_iterator subtype_begin() const { return ContainedTys; } |
319 | subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];} |
320 | ArrayRef<Type*> subtypes() const { |
321 | return makeArrayRef(subtype_begin(), subtype_end()); |
322 | } |
323 | |
324 | using subtype_reverse_iterator = std::reverse_iterator<subtype_iterator>; |
325 | |
326 | subtype_reverse_iterator subtype_rbegin() const { |
327 | return subtype_reverse_iterator(subtype_end()); |
328 | } |
329 | subtype_reverse_iterator subtype_rend() const { |
330 | return subtype_reverse_iterator(subtype_begin()); |
331 | } |
332 | |
333 | /// This method is used to implement the type iterator (defined at the end of |
334 | /// the file). For derived types, this returns the types 'contained' in the |
335 | /// derived type. |
336 | Type *getContainedType(unsigned i) const { |
337 | assert(i < NumContainedTys && "Index out of range!")((i < NumContainedTys && "Index out of range!") ? static_cast <void> (0) : __assert_fail ("i < NumContainedTys && \"Index out of range!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 337, __PRETTY_FUNCTION__)); |
338 | return ContainedTys[i]; |
339 | } |
340 | |
341 | /// Return the number of types in the derived type. |
342 | unsigned getNumContainedTypes() const { return NumContainedTys; } |
343 | |
344 | //===--------------------------------------------------------------------===// |
345 | // Helper methods corresponding to subclass methods. This forces a cast to |
346 | // the specified subclass and calls its accessor. "getVectorNumElements" (for |
347 | // example) is shorthand for cast<VectorType>(Ty)->getNumElements(). This is |
348 | // only intended to cover the core methods that are frequently used, helper |
349 | // methods should not be added here. |
350 | |
351 | inline unsigned getIntegerBitWidth() const; |
352 | |
353 | inline Type *getFunctionParamType(unsigned i) const; |
354 | inline unsigned getFunctionNumParams() const; |
355 | inline bool isFunctionVarArg() const; |
356 | |
357 | inline StringRef getStructName() const; |
358 | inline unsigned getStructNumElements() const; |
359 | inline Type *getStructElementType(unsigned N) const; |
360 | |
361 | inline Type *getSequentialElementType() const { |
362 | assert(isSequentialType(getTypeID()) && "Not a sequential type!")((isSequentialType(getTypeID()) && "Not a sequential type!" ) ? static_cast<void> (0) : __assert_fail ("isSequentialType(getTypeID()) && \"Not a sequential type!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 362, __PRETTY_FUNCTION__)); |
363 | return ContainedTys[0]; |
364 | } |
365 | |
366 | inline uint64_t getArrayNumElements() const; |
367 | |
368 | Type *getArrayElementType() const { |
369 | assert(getTypeID() == ArrayTyID)((getTypeID() == ArrayTyID) ? static_cast<void> (0) : __assert_fail ("getTypeID() == ArrayTyID", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 369, __PRETTY_FUNCTION__)); |
370 | return ContainedTys[0]; |
371 | } |
372 | |
373 | inline bool getVectorIsScalable() const; |
374 | inline unsigned getVectorNumElements() const; |
375 | inline ElementCount getVectorElementCount() const; |
376 | Type *getVectorElementType() const { |
377 | assert(getTypeID() == VectorTyID)((getTypeID() == VectorTyID) ? static_cast<void> (0) : __assert_fail ("getTypeID() == VectorTyID", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 377, __PRETTY_FUNCTION__)); |
378 | return ContainedTys[0]; |
379 | } |
380 | |
381 | Type *getPointerElementType() const { |
382 | assert(getTypeID() == PointerTyID)((getTypeID() == PointerTyID) ? static_cast<void> (0) : __assert_fail ("getTypeID() == PointerTyID", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 382, __PRETTY_FUNCTION__)); |
383 | return ContainedTys[0]; |
384 | } |
385 | |
386 | /// Given an integer or vector type, change the lane bitwidth to NewBitwidth, |
387 | /// whilst keeping the old number of lanes. |
388 | inline Type *getWithNewBitWidth(unsigned NewBitWidth) const; |
389 | |
390 | /// Given scalar/vector integer type, returns a type with elements twice as |
391 | /// wide as in the original type. For vectors, preserves element count. |
392 | inline Type *getExtendedType() const; |
393 | |
394 | /// Get the address space of this pointer or pointer vector type. |
395 | inline unsigned getPointerAddressSpace() const; |
396 | |
397 | //===--------------------------------------------------------------------===// |
398 | // Static members exported by the Type class itself. Useful for getting |
399 | // instances of Type. |
400 | // |
401 | |
402 | /// Return a type based on an identifier. |
403 | static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber); |
404 | |
405 | //===--------------------------------------------------------------------===// |
406 | // These are the builtin types that are always available. |
407 | // |
408 | static Type *getVoidTy(LLVMContext &C); |
409 | static Type *getLabelTy(LLVMContext &C); |
410 | static Type *getHalfTy(LLVMContext &C); |
411 | static Type *getFloatTy(LLVMContext &C); |
412 | static Type *getDoubleTy(LLVMContext &C); |
413 | static Type *getMetadataTy(LLVMContext &C); |
414 | static Type *getX86_FP80Ty(LLVMContext &C); |
415 | static Type *getFP128Ty(LLVMContext &C); |
416 | static Type *getPPC_FP128Ty(LLVMContext &C); |
417 | static Type *getX86_MMXTy(LLVMContext &C); |
418 | static Type *getTokenTy(LLVMContext &C); |
419 | static IntegerType *getIntNTy(LLVMContext &C, unsigned N); |
420 | static IntegerType *getInt1Ty(LLVMContext &C); |
421 | static IntegerType *getInt8Ty(LLVMContext &C); |
422 | static IntegerType *getInt16Ty(LLVMContext &C); |
423 | static IntegerType *getInt32Ty(LLVMContext &C); |
424 | static IntegerType *getInt64Ty(LLVMContext &C); |
425 | static IntegerType *getInt128Ty(LLVMContext &C); |
426 | template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) { |
427 | int noOfBits = sizeof(ScalarTy) * CHAR_BIT8; |
428 | if (std::is_integral<ScalarTy>::value) { |
429 | return (Type*) Type::getIntNTy(C, noOfBits); |
430 | } else if (std::is_floating_point<ScalarTy>::value) { |
431 | switch (noOfBits) { |
432 | case 32: |
433 | return Type::getFloatTy(C); |
434 | case 64: |
435 | return Type::getDoubleTy(C); |
436 | } |
437 | } |
438 | llvm_unreachable("Unsupported type in Type::getScalarTy")::llvm::llvm_unreachable_internal("Unsupported type in Type::getScalarTy" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h" , 438); |
439 | } |
440 | |
441 | //===--------------------------------------------------------------------===// |
442 | // Convenience methods for getting pointer types with one of the above builtin |
443 | // types as pointee. |
444 | // |
445 | static PointerType *getHalfPtrTy(LLVMContext &C, unsigned AS = 0); |
446 | static PointerType *getFloatPtrTy(LLVMContext &C, unsigned AS = 0); |
447 | static PointerType *getDoublePtrTy(LLVMContext &C, unsigned AS = 0); |
448 | static PointerType *getX86_FP80PtrTy(LLVMContext &C, unsigned AS = 0); |
449 | static PointerType *getFP128PtrTy(LLVMContext &C, unsigned AS = 0); |
450 | static PointerType *getPPC_FP128PtrTy(LLVMContext &C, unsigned AS = 0); |
451 | static PointerType *getX86_MMXPtrTy(LLVMContext &C, unsigned AS = 0); |
452 | static PointerType *getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS = 0); |
453 | static PointerType *getInt1PtrTy(LLVMContext &C, unsigned AS = 0); |
454 | static PointerType *getInt8PtrTy(LLVMContext &C, unsigned AS = 0); |
455 | static PointerType *getInt16PtrTy(LLVMContext &C, unsigned AS = 0); |
456 | static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0); |
457 | static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0); |
458 | |
459 | /// Return a pointer to the current type. This is equivalent to |
460 | /// PointerType::get(Foo, AddrSpace). |
461 | PointerType *getPointerTo(unsigned AddrSpace = 0) const; |
462 | |
463 | private: |
464 | /// Derived types like structures and arrays are sized iff all of the members |
465 | /// of the type are sized as well. Since asking for their size is relatively |
466 | /// uncommon, move this operation out-of-line. |
467 | bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const; |
468 | }; |
469 | |
470 | // Printing of types. |
471 | inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) { |
472 | T.print(OS); |
473 | return OS; |
474 | } |
475 | |
476 | // allow isa<PointerType>(x) to work without DerivedTypes.h included. |
477 | template <> struct isa_impl<PointerType, Type> { |
478 | static inline bool doit(const Type &Ty) { |
479 | return Ty.getTypeID() == Type::PointerTyID; |
480 | } |
481 | }; |
482 | |
483 | // Create wrappers for C Binding types (see CBindingWrapping.h). |
484 | DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)inline Type *unwrap(LLVMTypeRef P) { return reinterpret_cast< Type*>(P); } inline LLVMTypeRef wrap(const Type *P) { return reinterpret_cast<LLVMTypeRef>(const_cast<Type*>( P)); } template<typename T> inline T *unwrap(LLVMTypeRef P) { return cast<T>(unwrap(P)); } |
485 | |
486 | /* Specialized opaque type conversions. |
487 | */ |
488 | inline Type **unwrap(LLVMTypeRef* Tys) { |
489 | return reinterpret_cast<Type**>(Tys); |
490 | } |
491 | |
492 | inline LLVMTypeRef *wrap(Type **Tys) { |
493 | return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys)); |
494 | } |
495 | |
496 | } // end namespace llvm |
497 | |
498 | #endif // LLVM_IR_TYPE_H |
1 | //===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | // | ||||||
9 | // This file defines the IRBuilder class, which is used as a convenient way | ||||||
10 | // to create LLVM instructions with a consistent and simplified interface. | ||||||
11 | // | ||||||
12 | //===----------------------------------------------------------------------===// | ||||||
13 | |||||||
14 | #ifndef LLVM_IR_IRBUILDER_H | ||||||
15 | #define LLVM_IR_IRBUILDER_H | ||||||
16 | |||||||
17 | #include "llvm-c/Types.h" | ||||||
18 | #include "llvm/ADT/ArrayRef.h" | ||||||
19 | #include "llvm/ADT/None.h" | ||||||
20 | #include "llvm/ADT/StringRef.h" | ||||||
21 | #include "llvm/ADT/Twine.h" | ||||||
22 | #include "llvm/IR/BasicBlock.h" | ||||||
23 | #include "llvm/IR/Constant.h" | ||||||
24 | #include "llvm/IR/ConstantFolder.h" | ||||||
25 | #include "llvm/IR/Constants.h" | ||||||
26 | #include "llvm/IR/DataLayout.h" | ||||||
27 | #include "llvm/IR/DebugLoc.h" | ||||||
28 | #include "llvm/IR/DerivedTypes.h" | ||||||
29 | #include "llvm/IR/Function.h" | ||||||
30 | #include "llvm/IR/GlobalVariable.h" | ||||||
31 | #include "llvm/IR/InstrTypes.h" | ||||||
32 | #include "llvm/IR/Instruction.h" | ||||||
33 | #include "llvm/IR/Instructions.h" | ||||||
34 | #include "llvm/IR/IntrinsicInst.h" | ||||||
35 | #include "llvm/IR/LLVMContext.h" | ||||||
36 | #include "llvm/IR/Module.h" | ||||||
37 | #include "llvm/IR/Operator.h" | ||||||
38 | #include "llvm/IR/Type.h" | ||||||
39 | #include "llvm/IR/Value.h" | ||||||
40 | #include "llvm/IR/ValueHandle.h" | ||||||
41 | #include "llvm/Support/AtomicOrdering.h" | ||||||
42 | #include "llvm/Support/CBindingWrapping.h" | ||||||
43 | #include "llvm/Support/Casting.h" | ||||||
44 | #include <cassert> | ||||||
45 | #include <cstddef> | ||||||
46 | #include <cstdint> | ||||||
47 | #include <functional> | ||||||
48 | #include <utility> | ||||||
49 | |||||||
50 | namespace llvm { | ||||||
51 | |||||||
52 | class APInt; | ||||||
53 | class MDNode; | ||||||
54 | class Use; | ||||||
55 | |||||||
56 | /// This provides the default implementation of the IRBuilder | ||||||
57 | /// 'InsertHelper' method that is called whenever an instruction is created by | ||||||
58 | /// IRBuilder and needs to be inserted. | ||||||
59 | /// | ||||||
60 | /// By default, this inserts the instruction at the insertion point. | ||||||
61 | class IRBuilderDefaultInserter { | ||||||
62 | protected: | ||||||
63 | void InsertHelper(Instruction *I, const Twine &Name, | ||||||
64 | BasicBlock *BB, BasicBlock::iterator InsertPt) const { | ||||||
65 | if (BB) BB->getInstList().insert(InsertPt, I); | ||||||
66 | I->setName(Name); | ||||||
67 | } | ||||||
68 | }; | ||||||
69 | |||||||
70 | /// Provides an 'InsertHelper' that calls a user-provided callback after | ||||||
71 | /// performing the default insertion. | ||||||
72 | class IRBuilderCallbackInserter : IRBuilderDefaultInserter { | ||||||
73 | std::function<void(Instruction *)> Callback; | ||||||
74 | |||||||
75 | public: | ||||||
76 | IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback) | ||||||
77 | : Callback(std::move(Callback)) {} | ||||||
78 | |||||||
79 | protected: | ||||||
80 | void InsertHelper(Instruction *I, const Twine &Name, | ||||||
81 | BasicBlock *BB, BasicBlock::iterator InsertPt) const { | ||||||
82 | IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt); | ||||||
83 | Callback(I); | ||||||
84 | } | ||||||
85 | }; | ||||||
86 | |||||||
87 | /// Common base class shared among various IRBuilders. | ||||||
88 | class IRBuilderBase { | ||||||
89 | DebugLoc CurDbgLocation; | ||||||
90 | |||||||
91 | protected: | ||||||
92 | BasicBlock *BB; | ||||||
93 | BasicBlock::iterator InsertPt; | ||||||
94 | LLVMContext &Context; | ||||||
95 | |||||||
96 | MDNode *DefaultFPMathTag; | ||||||
97 | FastMathFlags FMF; | ||||||
98 | |||||||
99 | bool IsFPConstrained; | ||||||
100 | fp::ExceptionBehavior DefaultConstrainedExcept; | ||||||
101 | fp::RoundingMode DefaultConstrainedRounding; | ||||||
102 | |||||||
103 | ArrayRef<OperandBundleDef> DefaultOperandBundles; | ||||||
104 | |||||||
105 | public: | ||||||
106 | IRBuilderBase(LLVMContext &context, MDNode *FPMathTag = nullptr, | ||||||
107 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
108 | : Context(context), DefaultFPMathTag(FPMathTag), IsFPConstrained(false), | ||||||
109 | DefaultConstrainedExcept(fp::ebStrict), | ||||||
110 | DefaultConstrainedRounding(fp::rmDynamic), | ||||||
111 | DefaultOperandBundles(OpBundles) { | ||||||
112 | ClearInsertionPoint(); | ||||||
113 | } | ||||||
114 | |||||||
115 | //===--------------------------------------------------------------------===// | ||||||
116 | // Builder configuration methods | ||||||
117 | //===--------------------------------------------------------------------===// | ||||||
118 | |||||||
119 | /// Clear the insertion point: created instructions will not be | ||||||
120 | /// inserted into a block. | ||||||
121 | void ClearInsertionPoint() { | ||||||
122 | BB = nullptr; | ||||||
123 | InsertPt = BasicBlock::iterator(); | ||||||
124 | } | ||||||
125 | |||||||
126 | BasicBlock *GetInsertBlock() const { return BB; } | ||||||
127 | BasicBlock::iterator GetInsertPoint() const { return InsertPt; } | ||||||
128 | LLVMContext &getContext() const { return Context; } | ||||||
129 | |||||||
130 | /// This specifies that created instructions should be appended to the | ||||||
131 | /// end of the specified block. | ||||||
132 | void SetInsertPoint(BasicBlock *TheBB) { | ||||||
133 | BB = TheBB; | ||||||
134 | InsertPt = BB->end(); | ||||||
135 | } | ||||||
136 | |||||||
137 | /// This specifies that created instructions should be inserted before | ||||||
138 | /// the specified instruction. | ||||||
139 | void SetInsertPoint(Instruction *I) { | ||||||
140 | BB = I->getParent(); | ||||||
141 | InsertPt = I->getIterator(); | ||||||
142 | assert(InsertPt != BB->end() && "Can't read debug loc from end()")((InsertPt != BB->end() && "Can't read debug loc from end()" ) ? static_cast<void> (0) : __assert_fail ("InsertPt != BB->end() && \"Can't read debug loc from end()\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 142, __PRETTY_FUNCTION__)); | ||||||
143 | SetCurrentDebugLocation(I->getDebugLoc()); | ||||||
144 | } | ||||||
145 | |||||||
146 | /// This specifies that created instructions should be inserted at the | ||||||
147 | /// specified point. | ||||||
148 | void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) { | ||||||
149 | BB = TheBB; | ||||||
150 | InsertPt = IP; | ||||||
151 | if (IP != TheBB->end()) | ||||||
152 | SetCurrentDebugLocation(IP->getDebugLoc()); | ||||||
153 | } | ||||||
154 | |||||||
155 | /// Set location information used by debugging information. | ||||||
156 | void SetCurrentDebugLocation(DebugLoc L) { CurDbgLocation = std::move(L); } | ||||||
157 | |||||||
158 | /// Get location information used by debugging information. | ||||||
159 | const DebugLoc &getCurrentDebugLocation() const { return CurDbgLocation; } | ||||||
160 | |||||||
161 | /// If this builder has a current debug location, set it on the | ||||||
162 | /// specified instruction. | ||||||
163 | void SetInstDebugLocation(Instruction *I) const { | ||||||
164 | if (CurDbgLocation) | ||||||
165 | I->setDebugLoc(CurDbgLocation); | ||||||
166 | } | ||||||
167 | |||||||
168 | /// Get the return type of the current function that we're emitting | ||||||
169 | /// into. | ||||||
170 | Type *getCurrentFunctionReturnType() const; | ||||||
171 | |||||||
172 | /// InsertPoint - A saved insertion point. | ||||||
173 | class InsertPoint { | ||||||
174 | BasicBlock *Block = nullptr; | ||||||
175 | BasicBlock::iterator Point; | ||||||
176 | |||||||
177 | public: | ||||||
178 | /// Creates a new insertion point which doesn't point to anything. | ||||||
179 | InsertPoint() = default; | ||||||
180 | |||||||
181 | /// Creates a new insertion point at the given location. | ||||||
182 | InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint) | ||||||
183 | : Block(InsertBlock), Point(InsertPoint) {} | ||||||
184 | |||||||
185 | /// Returns true if this insert point is set. | ||||||
186 | bool isSet() const { return (Block != nullptr); } | ||||||
187 | |||||||
188 | BasicBlock *getBlock() const { return Block; } | ||||||
189 | BasicBlock::iterator getPoint() const { return Point; } | ||||||
190 | }; | ||||||
191 | |||||||
192 | /// Returns the current insert point. | ||||||
193 | InsertPoint saveIP() const { | ||||||
194 | return InsertPoint(GetInsertBlock(), GetInsertPoint()); | ||||||
195 | } | ||||||
196 | |||||||
197 | /// Returns the current insert point, clearing it in the process. | ||||||
198 | InsertPoint saveAndClearIP() { | ||||||
199 | InsertPoint IP(GetInsertBlock(), GetInsertPoint()); | ||||||
200 | ClearInsertionPoint(); | ||||||
201 | return IP; | ||||||
202 | } | ||||||
203 | |||||||
204 | /// Sets the current insert point to a previously-saved location. | ||||||
205 | void restoreIP(InsertPoint IP) { | ||||||
206 | if (IP.isSet()) | ||||||
207 | SetInsertPoint(IP.getBlock(), IP.getPoint()); | ||||||
208 | else | ||||||
209 | ClearInsertionPoint(); | ||||||
210 | } | ||||||
211 | |||||||
212 | /// Get the floating point math metadata being used. | ||||||
213 | MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; } | ||||||
214 | |||||||
215 | /// Get the flags to be applied to created floating point ops | ||||||
216 | FastMathFlags getFastMathFlags() const { return FMF; } | ||||||
217 | |||||||
218 | /// Clear the fast-math flags. | ||||||
219 | void clearFastMathFlags() { FMF.clear(); } | ||||||
220 | |||||||
221 | /// Set the floating point math metadata to be used. | ||||||
222 | void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; } | ||||||
223 | |||||||
224 | /// Set the fast-math flags to be used with generated fp-math operators | ||||||
225 | void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; } | ||||||
226 | |||||||
227 | /// Enable/Disable use of constrained floating point math. When | ||||||
228 | /// enabled the CreateF<op>() calls instead create constrained | ||||||
229 | /// floating point intrinsic calls. Fast math flags are unaffected | ||||||
230 | /// by this setting. | ||||||
231 | void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; } | ||||||
232 | |||||||
233 | /// Query for the use of constrained floating point math | ||||||
234 | bool getIsFPConstrained() { return IsFPConstrained; } | ||||||
235 | |||||||
236 | /// Set the exception handling to be used with constrained floating point | ||||||
237 | void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) { | ||||||
238 | DefaultConstrainedExcept = NewExcept; | ||||||
239 | } | ||||||
240 | |||||||
241 | /// Set the rounding mode handling to be used with constrained floating point | ||||||
242 | void setDefaultConstrainedRounding(fp::RoundingMode NewRounding) { | ||||||
243 | DefaultConstrainedRounding = NewRounding; | ||||||
244 | } | ||||||
245 | |||||||
246 | /// Get the exception handling used with constrained floating point | ||||||
247 | fp::ExceptionBehavior getDefaultConstrainedExcept() { | ||||||
248 | return DefaultConstrainedExcept; | ||||||
249 | } | ||||||
250 | |||||||
251 | /// Get the rounding mode handling used with constrained floating point | ||||||
252 | fp::RoundingMode getDefaultConstrainedRounding() { | ||||||
253 | return DefaultConstrainedRounding; | ||||||
254 | } | ||||||
255 | |||||||
256 | void setConstrainedFPFunctionAttr() { | ||||||
257 | assert(BB && "Must have a basic block to set any function attributes!")((BB && "Must have a basic block to set any function attributes!" ) ? static_cast<void> (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 257, __PRETTY_FUNCTION__)); | ||||||
258 | |||||||
259 | Function *F = BB->getParent(); | ||||||
260 | if (!F->hasFnAttribute(Attribute::StrictFP)) { | ||||||
261 | F->addFnAttr(Attribute::StrictFP); | ||||||
262 | } | ||||||
263 | } | ||||||
264 | |||||||
265 | void setConstrainedFPCallAttr(CallInst *I) { | ||||||
266 | if (!I->hasFnAttr(Attribute::StrictFP)) | ||||||
267 | I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP); | ||||||
268 | } | ||||||
269 | |||||||
270 | //===--------------------------------------------------------------------===// | ||||||
271 | // RAII helpers. | ||||||
272 | //===--------------------------------------------------------------------===// | ||||||
273 | |||||||
274 | // RAII object that stores the current insertion point and restores it | ||||||
275 | // when the object is destroyed. This includes the debug location. | ||||||
276 | class InsertPointGuard { | ||||||
277 | IRBuilderBase &Builder; | ||||||
278 | AssertingVH<BasicBlock> Block; | ||||||
279 | BasicBlock::iterator Point; | ||||||
280 | DebugLoc DbgLoc; | ||||||
281 | |||||||
282 | public: | ||||||
283 | InsertPointGuard(IRBuilderBase &B) | ||||||
284 | : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()), | ||||||
285 | DbgLoc(B.getCurrentDebugLocation()) {} | ||||||
286 | |||||||
287 | InsertPointGuard(const InsertPointGuard &) = delete; | ||||||
288 | InsertPointGuard &operator=(const InsertPointGuard &) = delete; | ||||||
289 | |||||||
290 | ~InsertPointGuard() { | ||||||
291 | Builder.restoreIP(InsertPoint(Block, Point)); | ||||||
292 | Builder.SetCurrentDebugLocation(DbgLoc); | ||||||
293 | } | ||||||
294 | }; | ||||||
295 | |||||||
296 | // RAII object that stores the current fast math settings and restores | ||||||
297 | // them when the object is destroyed. | ||||||
298 | class FastMathFlagGuard { | ||||||
299 | IRBuilderBase &Builder; | ||||||
300 | FastMathFlags FMF; | ||||||
301 | MDNode *FPMathTag; | ||||||
302 | |||||||
303 | public: | ||||||
304 | FastMathFlagGuard(IRBuilderBase &B) | ||||||
305 | : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag) {} | ||||||
306 | |||||||
307 | FastMathFlagGuard(const FastMathFlagGuard &) = delete; | ||||||
308 | FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete; | ||||||
309 | |||||||
310 | ~FastMathFlagGuard() { | ||||||
311 | Builder.FMF = FMF; | ||||||
312 | Builder.DefaultFPMathTag = FPMathTag; | ||||||
313 | } | ||||||
314 | }; | ||||||
315 | |||||||
316 | //===--------------------------------------------------------------------===// | ||||||
317 | // Miscellaneous creation methods. | ||||||
318 | //===--------------------------------------------------------------------===// | ||||||
319 | |||||||
320 | /// Make a new global variable with initializer type i8* | ||||||
321 | /// | ||||||
322 | /// Make a new global variable with an initializer that has array of i8 type | ||||||
323 | /// filled in with the null terminated string value specified. The new global | ||||||
324 | /// variable will be marked mergable with any others of the same contents. If | ||||||
325 | /// Name is specified, it is the name of the global variable created. | ||||||
326 | GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "", | ||||||
327 | unsigned AddressSpace = 0); | ||||||
328 | |||||||
329 | /// Get a constant value representing either true or false. | ||||||
330 | ConstantInt *getInt1(bool V) { | ||||||
331 | return ConstantInt::get(getInt1Ty(), V); | ||||||
332 | } | ||||||
333 | |||||||
334 | /// Get the constant value for i1 true. | ||||||
335 | ConstantInt *getTrue() { | ||||||
336 | return ConstantInt::getTrue(Context); | ||||||
337 | } | ||||||
338 | |||||||
339 | /// Get the constant value for i1 false. | ||||||
340 | ConstantInt *getFalse() { | ||||||
341 | return ConstantInt::getFalse(Context); | ||||||
342 | } | ||||||
343 | |||||||
344 | /// Get a constant 8-bit value. | ||||||
345 | ConstantInt *getInt8(uint8_t C) { | ||||||
346 | return ConstantInt::get(getInt8Ty(), C); | ||||||
347 | } | ||||||
348 | |||||||
349 | /// Get a constant 16-bit value. | ||||||
350 | ConstantInt *getInt16(uint16_t C) { | ||||||
351 | return ConstantInt::get(getInt16Ty(), C); | ||||||
352 | } | ||||||
353 | |||||||
354 | /// Get a constant 32-bit value. | ||||||
355 | ConstantInt *getInt32(uint32_t C) { | ||||||
356 | return ConstantInt::get(getInt32Ty(), C); | ||||||
357 | } | ||||||
358 | |||||||
359 | /// Get a constant 64-bit value. | ||||||
360 | ConstantInt *getInt64(uint64_t C) { | ||||||
361 | return ConstantInt::get(getInt64Ty(), C); | ||||||
362 | } | ||||||
363 | |||||||
364 | /// Get a constant N-bit value, zero extended or truncated from | ||||||
365 | /// a 64-bit value. | ||||||
366 | ConstantInt *getIntN(unsigned N, uint64_t C) { | ||||||
367 | return ConstantInt::get(getIntNTy(N), C); | ||||||
368 | } | ||||||
369 | |||||||
370 | /// Get a constant integer value. | ||||||
371 | ConstantInt *getInt(const APInt &AI) { | ||||||
372 | return ConstantInt::get(Context, AI); | ||||||
373 | } | ||||||
374 | |||||||
375 | //===--------------------------------------------------------------------===// | ||||||
376 | // Type creation methods | ||||||
377 | //===--------------------------------------------------------------------===// | ||||||
378 | |||||||
379 | /// Fetch the type representing a single bit | ||||||
380 | IntegerType *getInt1Ty() { | ||||||
381 | return Type::getInt1Ty(Context); | ||||||
382 | } | ||||||
383 | |||||||
384 | /// Fetch the type representing an 8-bit integer. | ||||||
385 | IntegerType *getInt8Ty() { | ||||||
386 | return Type::getInt8Ty(Context); | ||||||
387 | } | ||||||
388 | |||||||
389 | /// Fetch the type representing a 16-bit integer. | ||||||
390 | IntegerType *getInt16Ty() { | ||||||
391 | return Type::getInt16Ty(Context); | ||||||
392 | } | ||||||
393 | |||||||
394 | /// Fetch the type representing a 32-bit integer. | ||||||
395 | IntegerType *getInt32Ty() { | ||||||
396 | return Type::getInt32Ty(Context); | ||||||
397 | } | ||||||
398 | |||||||
399 | /// Fetch the type representing a 64-bit integer. | ||||||
400 | IntegerType *getInt64Ty() { | ||||||
401 | return Type::getInt64Ty(Context); | ||||||
402 | } | ||||||
403 | |||||||
404 | /// Fetch the type representing a 128-bit integer. | ||||||
405 | IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); } | ||||||
406 | |||||||
407 | /// Fetch the type representing an N-bit integer. | ||||||
408 | IntegerType *getIntNTy(unsigned N) { | ||||||
409 | return Type::getIntNTy(Context, N); | ||||||
410 | } | ||||||
411 | |||||||
412 | /// Fetch the type representing a 16-bit floating point value. | ||||||
413 | Type *getHalfTy() { | ||||||
414 | return Type::getHalfTy(Context); | ||||||
415 | } | ||||||
416 | |||||||
417 | /// Fetch the type representing a 32-bit floating point value. | ||||||
418 | Type *getFloatTy() { | ||||||
419 | return Type::getFloatTy(Context); | ||||||
420 | } | ||||||
421 | |||||||
422 | /// Fetch the type representing a 64-bit floating point value. | ||||||
423 | Type *getDoubleTy() { | ||||||
424 | return Type::getDoubleTy(Context); | ||||||
425 | } | ||||||
426 | |||||||
427 | /// Fetch the type representing void. | ||||||
428 | Type *getVoidTy() { | ||||||
429 | return Type::getVoidTy(Context); | ||||||
430 | } | ||||||
431 | |||||||
432 | /// Fetch the type representing a pointer to an 8-bit integer value. | ||||||
433 | PointerType *getInt8PtrTy(unsigned AddrSpace = 0) { | ||||||
434 | return Type::getInt8PtrTy(Context, AddrSpace); | ||||||
435 | } | ||||||
436 | |||||||
437 | /// Fetch the type representing a pointer to an integer value. | ||||||
438 | IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) { | ||||||
439 | return DL.getIntPtrType(Context, AddrSpace); | ||||||
440 | } | ||||||
441 | |||||||
442 | //===--------------------------------------------------------------------===// | ||||||
443 | // Intrinsic creation methods | ||||||
444 | //===--------------------------------------------------------------------===// | ||||||
445 | |||||||
446 | /// Create and insert a memset to the specified pointer and the | ||||||
447 | /// specified value. | ||||||
448 | /// | ||||||
449 | /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is | ||||||
450 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
451 | /// and noalias tags. | ||||||
452 | CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, | ||||||
453 | MaybeAlign Align, bool isVolatile = false, | ||||||
454 | MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
455 | MDNode *NoAliasTag = nullptr) { | ||||||
456 | return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, | ||||||
457 | TBAATag, ScopeTag, NoAliasTag); | ||||||
458 | } | ||||||
459 | |||||||
460 | CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align, | ||||||
461 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
462 | MDNode *ScopeTag = nullptr, | ||||||
463 | MDNode *NoAliasTag = nullptr); | ||||||
464 | |||||||
465 | /// Create and insert an element unordered-atomic memset of the region of | ||||||
466 | /// memory starting at the given pointer to the given value. | ||||||
467 | /// | ||||||
468 | /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is | ||||||
469 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
470 | /// and noalias tags. | ||||||
471 | /// FIXME: Remove this function once transition to Align is over. | ||||||
472 | /// Use the version that takes Align instead of this one. | ||||||
473 | LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
474 | CallInst *CreateElementUnorderedAtomicMemSet(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
475 | Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
476 | uint32_t ElementSize, MDNode *TBAATag = nullptr,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
477 | MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
478 | "Use the version that takes Align instead of this one")CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) { | ||||||
479 | return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size), | ||||||
480 | Align(Alignment), ElementSize, | ||||||
481 | TBAATag, ScopeTag, NoAliasTag); | ||||||
482 | } | ||||||
483 | |||||||
484 | CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, | ||||||
485 | uint64_t Size, Align Alignment, | ||||||
486 | uint32_t ElementSize, | ||||||
487 | MDNode *TBAATag = nullptr, | ||||||
488 | MDNode *ScopeTag = nullptr, | ||||||
489 | MDNode *NoAliasTag = nullptr) { | ||||||
490 | return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size), | ||||||
491 | Align(Alignment), ElementSize, | ||||||
492 | TBAATag, ScopeTag, NoAliasTag); | ||||||
493 | } | ||||||
494 | |||||||
495 | /// FIXME: Remove this function once transition to Align is over. | ||||||
496 | /// Use the version that takes Align instead of this one. | ||||||
497 | LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, unsigned Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
498 | CallInst *CreateElementUnorderedAtomicMemSet(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, unsigned Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
499 | Value *Ptr, Value *Val, Value *Size, unsigned Alignment,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, unsigned Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
500 | uint32_t ElementSize, MDNode *TBAATag = nullptr,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, unsigned Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
501 | MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, unsigned Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) | ||||||
502 | "Use the version that takes Align instead of this one")CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value *Val, Value *Size, unsigned Alignment, uint32_t ElementSize, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one" ))) { | ||||||
503 | return CreateElementUnorderedAtomicMemSet(Ptr, Val, Size, Align(Alignment), | ||||||
504 | ElementSize, TBAATag, ScopeTag, | ||||||
505 | NoAliasTag); | ||||||
506 | } | ||||||
507 | |||||||
508 | CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, | ||||||
509 | Value *Size, Align Alignment, | ||||||
510 | uint32_t ElementSize, | ||||||
511 | MDNode *TBAATag = nullptr, | ||||||
512 | MDNode *ScopeTag = nullptr, | ||||||
513 | MDNode *NoAliasTag = nullptr); | ||||||
514 | |||||||
515 | /// Create and insert a memcpy between the specified pointers. | ||||||
516 | /// | ||||||
517 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
518 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
519 | /// and noalias tags. | ||||||
520 | /// FIXME: Remove this function once transition to Align is over. | ||||||
521 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
522 | LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
523 | CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
524 | unsigned SrcAlign, uint64_t Size,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
525 | bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
526 | MDNode *TBAAStructTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
527 | MDNode *ScopeTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
528 | MDNode *NoAliasTag = nullptr),CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
529 | "Use the version that takes MaybeAlign instead")CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) { | ||||||
530 | return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), | ||||||
531 | getInt64(Size), isVolatile, TBAATag, TBAAStructTag, | ||||||
532 | ScopeTag, NoAliasTag); | ||||||
533 | } | ||||||
534 | |||||||
535 | CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, | ||||||
536 | MaybeAlign SrcAlign, uint64_t Size, | ||||||
537 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
538 | MDNode *TBAAStructTag = nullptr, | ||||||
539 | MDNode *ScopeTag = nullptr, | ||||||
540 | MDNode *NoAliasTag = nullptr) { | ||||||
541 | return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size), | ||||||
542 | isVolatile, TBAATag, TBAAStructTag, ScopeTag, | ||||||
543 | NoAliasTag); | ||||||
544 | } | ||||||
545 | |||||||
546 | /// FIXME: Remove this function once transition to Align is over. | ||||||
547 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
548 | LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
549 | CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
550 | unsigned SrcAlign, Value *Size,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
551 | bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
552 | MDNode *TBAAStructTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
553 | MDNode *ScopeTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
554 | MDNode *NoAliasTag = nullptr),CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )) | ||||||
555 | "Use the version that takes MaybeAlign instead")CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value * Src, unsigned SrcAlign, Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__ ((deprecated("Use the version that takes MaybeAlign instead") )); | ||||||
556 | CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, | ||||||
557 | MaybeAlign SrcAlign, Value *Size, | ||||||
558 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
559 | MDNode *TBAAStructTag = nullptr, | ||||||
560 | MDNode *ScopeTag = nullptr, | ||||||
561 | MDNode *NoAliasTag = nullptr); | ||||||
562 | |||||||
563 | /// Create and insert an element unordered-atomic memcpy between the | ||||||
564 | /// specified pointers. | ||||||
565 | /// | ||||||
566 | /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively. | ||||||
567 | /// | ||||||
568 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
569 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
570 | /// and noalias tags. | ||||||
571 | CallInst *CreateElementUnorderedAtomicMemCpy( | ||||||
572 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, | ||||||
573 | uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
574 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
575 | MDNode *NoAliasTag = nullptr) { | ||||||
576 | return CreateElementUnorderedAtomicMemCpy( | ||||||
577 | Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag, | ||||||
578 | TBAAStructTag, ScopeTag, NoAliasTag); | ||||||
579 | } | ||||||
580 | |||||||
581 | CallInst *CreateElementUnorderedAtomicMemCpy( | ||||||
582 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, | ||||||
583 | uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
584 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
585 | MDNode *NoAliasTag = nullptr); | ||||||
586 | |||||||
587 | /// Create and insert a memmove between the specified | ||||||
588 | /// pointers. | ||||||
589 | /// | ||||||
590 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
591 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
592 | /// and noalias tags. | ||||||
593 | /// FIXME: Remove this function once transition to Align is over. | ||||||
594 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
595 | LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
596 | CallInst *CreateMemMove(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
597 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
598 | uint64_t Size, bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
599 | MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
600 | "Use the version that takes MaybeAlign")CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) { | ||||||
601 | return CreateMemMove(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), | ||||||
602 | getInt64(Size), isVolatile, TBAATag, ScopeTag, | ||||||
603 | NoAliasTag); | ||||||
604 | } | ||||||
605 | CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, | ||||||
606 | MaybeAlign SrcAlign, uint64_t Size, | ||||||
607 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
608 | MDNode *ScopeTag = nullptr, | ||||||
609 | MDNode *NoAliasTag = nullptr) { | ||||||
610 | return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size), | ||||||
611 | isVolatile, TBAATag, ScopeTag, NoAliasTag); | ||||||
612 | } | ||||||
613 | /// FIXME: Remove this function once transition to Align is over. | ||||||
614 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
615 | LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
616 | CallInst *CreateMemMove(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
617 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
618 | Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
619 | MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) | ||||||
620 | "Use the version that takes MaybeAlign")CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, bool isVolatile = false , MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign" ))) { | ||||||
621 | return CreateMemMove(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign), | ||||||
622 | Size, isVolatile, TBAATag, ScopeTag, NoAliasTag); | ||||||
623 | } | ||||||
624 | CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src, | ||||||
625 | MaybeAlign SrcAlign, Value *Size, | ||||||
626 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
627 | MDNode *ScopeTag = nullptr, | ||||||
628 | MDNode *NoAliasTag = nullptr); | ||||||
629 | |||||||
630 | /// \brief Create and insert an element unordered-atomic memmove between the | ||||||
631 | /// specified pointers. | ||||||
632 | /// | ||||||
633 | /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, | ||||||
634 | /// respectively. | ||||||
635 | /// | ||||||
636 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
637 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
638 | /// and noalias tags. | ||||||
639 | CallInst *CreateElementUnorderedAtomicMemMove( | ||||||
640 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, | ||||||
641 | uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
642 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
643 | MDNode *NoAliasTag = nullptr) { | ||||||
644 | return CreateElementUnorderedAtomicMemMove( | ||||||
645 | Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag, | ||||||
646 | TBAAStructTag, ScopeTag, NoAliasTag); | ||||||
647 | } | ||||||
648 | |||||||
649 | CallInst *CreateElementUnorderedAtomicMemMove( | ||||||
650 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, | ||||||
651 | uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
652 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
653 | MDNode *NoAliasTag = nullptr); | ||||||
654 | |||||||
655 | /// Create a vector fadd reduction intrinsic of the source vector. | ||||||
656 | /// The first parameter is a scalar accumulator value for ordered reductions. | ||||||
657 | CallInst *CreateFAddReduce(Value *Acc, Value *Src); | ||||||
658 | |||||||
659 | /// Create a vector fmul reduction intrinsic of the source vector. | ||||||
660 | /// The first parameter is a scalar accumulator value for ordered reductions. | ||||||
661 | CallInst *CreateFMulReduce(Value *Acc, Value *Src); | ||||||
662 | |||||||
663 | /// Create a vector int add reduction intrinsic of the source vector. | ||||||
664 | CallInst *CreateAddReduce(Value *Src); | ||||||
665 | |||||||
666 | /// Create a vector int mul reduction intrinsic of the source vector. | ||||||
667 | CallInst *CreateMulReduce(Value *Src); | ||||||
668 | |||||||
669 | /// Create a vector int AND reduction intrinsic of the source vector. | ||||||
670 | CallInst *CreateAndReduce(Value *Src); | ||||||
671 | |||||||
672 | /// Create a vector int OR reduction intrinsic of the source vector. | ||||||
673 | CallInst *CreateOrReduce(Value *Src); | ||||||
674 | |||||||
675 | /// Create a vector int XOR reduction intrinsic of the source vector. | ||||||
676 | CallInst *CreateXorReduce(Value *Src); | ||||||
677 | |||||||
678 | /// Create a vector integer max reduction intrinsic of the source | ||||||
679 | /// vector. | ||||||
680 | CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false); | ||||||
681 | |||||||
682 | /// Create a vector integer min reduction intrinsic of the source | ||||||
683 | /// vector. | ||||||
684 | CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false); | ||||||
685 | |||||||
686 | /// Create a vector float max reduction intrinsic of the source | ||||||
687 | /// vector. | ||||||
688 | CallInst *CreateFPMaxReduce(Value *Src, bool NoNaN = false); | ||||||
689 | |||||||
690 | /// Create a vector float min reduction intrinsic of the source | ||||||
691 | /// vector. | ||||||
692 | CallInst *CreateFPMinReduce(Value *Src, bool NoNaN = false); | ||||||
693 | |||||||
694 | /// Create a lifetime.start intrinsic. | ||||||
695 | /// | ||||||
696 | /// If the pointer isn't i8* it will be converted. | ||||||
697 | CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr); | ||||||
698 | |||||||
699 | /// Create a lifetime.end intrinsic. | ||||||
700 | /// | ||||||
701 | /// If the pointer isn't i8* it will be converted. | ||||||
702 | CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr); | ||||||
703 | |||||||
704 | /// Create a call to invariant.start intrinsic. | ||||||
705 | /// | ||||||
706 | /// If the pointer isn't i8* it will be converted. | ||||||
707 | CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr); | ||||||
708 | |||||||
709 | /// Create a call to Masked Load intrinsic | ||||||
710 | CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask, | ||||||
711 | Value *PassThru = nullptr, const Twine &Name = ""); | ||||||
712 | |||||||
713 | /// Create a call to Masked Store intrinsic | ||||||
714 | CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, | ||||||
715 | Value *Mask); | ||||||
716 | |||||||
717 | /// Create a call to Masked Gather intrinsic | ||||||
718 | CallInst *CreateMaskedGather(Value *Ptrs, unsigned Align, | ||||||
719 | Value *Mask = nullptr, | ||||||
720 | Value *PassThru = nullptr, | ||||||
721 | const Twine& Name = ""); | ||||||
722 | |||||||
723 | /// Create a call to Masked Scatter intrinsic | ||||||
724 | CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, unsigned Align, | ||||||
725 | Value *Mask = nullptr); | ||||||
726 | |||||||
727 | /// Create an assume intrinsic call that allows the optimizer to | ||||||
728 | /// assume that the provided condition will be true. | ||||||
729 | CallInst *CreateAssumption(Value *Cond); | ||||||
730 | |||||||
731 | /// Create a call to the experimental.gc.statepoint intrinsic to | ||||||
732 | /// start a new statepoint sequence. | ||||||
733 | CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, | ||||||
734 | Value *ActualCallee, | ||||||
735 | ArrayRef<Value *> CallArgs, | ||||||
736 | ArrayRef<Value *> DeoptArgs, | ||||||
737 | ArrayRef<Value *> GCArgs, | ||||||
738 | const Twine &Name = ""); | ||||||
739 | |||||||
740 | /// Create a call to the experimental.gc.statepoint intrinsic to | ||||||
741 | /// start a new statepoint sequence. | ||||||
742 | CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, | ||||||
743 | Value *ActualCallee, uint32_t Flags, | ||||||
744 | ArrayRef<Use> CallArgs, | ||||||
745 | ArrayRef<Use> TransitionArgs, | ||||||
746 | ArrayRef<Use> DeoptArgs, | ||||||
747 | ArrayRef<Value *> GCArgs, | ||||||
748 | const Twine &Name = ""); | ||||||
749 | |||||||
750 | /// Conveninence function for the common case when CallArgs are filled | ||||||
751 | /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be | ||||||
752 | /// .get()'ed to get the Value pointer. | ||||||
753 | CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, | ||||||
754 | Value *ActualCallee, ArrayRef<Use> CallArgs, | ||||||
755 | ArrayRef<Value *> DeoptArgs, | ||||||
756 | ArrayRef<Value *> GCArgs, | ||||||
757 | const Twine &Name = ""); | ||||||
758 | |||||||
759 | /// Create an invoke to the experimental.gc.statepoint intrinsic to | ||||||
760 | /// start a new statepoint sequence. | ||||||
761 | InvokeInst * | ||||||
762 | CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, | ||||||
763 | Value *ActualInvokee, BasicBlock *NormalDest, | ||||||
764 | BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs, | ||||||
765 | ArrayRef<Value *> DeoptArgs, | ||||||
766 | ArrayRef<Value *> GCArgs, const Twine &Name = ""); | ||||||
767 | |||||||
768 | /// Create an invoke to the experimental.gc.statepoint intrinsic to | ||||||
769 | /// start a new statepoint sequence. | ||||||
770 | InvokeInst *CreateGCStatepointInvoke( | ||||||
771 | uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, | ||||||
772 | BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, | ||||||
773 | ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs, | ||||||
774 | ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, | ||||||
775 | const Twine &Name = ""); | ||||||
776 | |||||||
777 | // Convenience function for the common case when CallArgs are filled in using | ||||||
778 | // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to | ||||||
779 | // get the Value *. | ||||||
780 | InvokeInst * | ||||||
781 | CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, | ||||||
782 | Value *ActualInvokee, BasicBlock *NormalDest, | ||||||
783 | BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, | ||||||
784 | ArrayRef<Value *> DeoptArgs, | ||||||
785 | ArrayRef<Value *> GCArgs, const Twine &Name = ""); | ||||||
786 | |||||||
787 | /// Create a call to the experimental.gc.result intrinsic to extract | ||||||
788 | /// the result from a call wrapped in a statepoint. | ||||||
789 | CallInst *CreateGCResult(Instruction *Statepoint, | ||||||
790 | Type *ResultType, | ||||||
791 | const Twine &Name = ""); | ||||||
792 | |||||||
793 | /// Create a call to the experimental.gc.relocate intrinsics to | ||||||
794 | /// project the relocated value of one pointer from the statepoint. | ||||||
795 | CallInst *CreateGCRelocate(Instruction *Statepoint, | ||||||
796 | int BaseOffset, | ||||||
797 | int DerivedOffset, | ||||||
798 | Type *ResultType, | ||||||
799 | const Twine &Name = ""); | ||||||
800 | |||||||
801 | /// Create a call to intrinsic \p ID with 1 operand which is mangled on its | ||||||
802 | /// type. | ||||||
803 | CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, | ||||||
804 | Instruction *FMFSource = nullptr, | ||||||
805 | const Twine &Name = ""); | ||||||
806 | |||||||
807 | /// Create a call to intrinsic \p ID with 2 operands which is mangled on the | ||||||
808 | /// first type. | ||||||
809 | CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, | ||||||
810 | Instruction *FMFSource = nullptr, | ||||||
811 | const Twine &Name = ""); | ||||||
812 | |||||||
813 | /// Create a call to intrinsic \p ID with \p args, mangled using \p Types. If | ||||||
814 | /// \p FMFSource is provided, copy fast-math-flags from that instruction to | ||||||
815 | /// the intrinsic. | ||||||
816 | CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types, | ||||||
817 | ArrayRef<Value *> Args, | ||||||
818 | Instruction *FMFSource = nullptr, | ||||||
819 | const Twine &Name = ""); | ||||||
820 | |||||||
821 | /// Create call to the minnum intrinsic. | ||||||
822 | CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
823 | return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name); | ||||||
824 | } | ||||||
825 | |||||||
826 | /// Create call to the maxnum intrinsic. | ||||||
827 | CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
828 | return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name); | ||||||
829 | } | ||||||
830 | |||||||
831 | /// Create call to the minimum intrinsic. | ||||||
832 | CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
833 | return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name); | ||||||
834 | } | ||||||
835 | |||||||
836 | /// Create call to the maximum intrinsic. | ||||||
837 | CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
838 | return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name); | ||||||
839 | } | ||||||
840 | |||||||
841 | private: | ||||||
842 | /// Create a call to a masked intrinsic with given Id. | ||||||
843 | CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops, | ||||||
844 | ArrayRef<Type *> OverloadedTypes, | ||||||
845 | const Twine &Name = ""); | ||||||
846 | |||||||
847 | Value *getCastedInt8PtrValue(Value *Ptr); | ||||||
848 | }; | ||||||
849 | |||||||
850 | /// This provides a uniform API for creating instructions and inserting | ||||||
851 | /// them into a basic block: either at the end of a BasicBlock, or at a specific | ||||||
852 | /// iterator location in a block. | ||||||
853 | /// | ||||||
854 | /// Note that the builder does not expose the full generality of LLVM | ||||||
855 | /// instructions. For access to extra instruction properties, use the mutators | ||||||
856 | /// (e.g. setVolatile) on the instructions after they have been | ||||||
857 | /// created. Convenience state exists to specify fast-math flags and fp-math | ||||||
858 | /// tags. | ||||||
859 | /// | ||||||
860 | /// The first template argument specifies a class to use for creating constants. | ||||||
861 | /// This defaults to creating minimally folded constants. The second template | ||||||
862 | /// argument allows clients to specify custom insertion hooks that are called on | ||||||
863 | /// every newly created insertion. | ||||||
864 | template <typename T = ConstantFolder, | ||||||
865 | typename Inserter = IRBuilderDefaultInserter> | ||||||
866 | class IRBuilder : public IRBuilderBase, public Inserter { | ||||||
867 | T Folder; | ||||||
868 | |||||||
869 | public: | ||||||
870 | IRBuilder(LLVMContext &C, const T &F, Inserter I = Inserter(), | ||||||
871 | MDNode *FPMathTag = nullptr, | ||||||
872 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
873 | : IRBuilderBase(C, FPMathTag, OpBundles), Inserter(std::move(I)), | ||||||
874 | Folder(F) {} | ||||||
875 | |||||||
876 | explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr, | ||||||
877 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
878 | : IRBuilderBase(C, FPMathTag, OpBundles) {} | ||||||
879 | |||||||
880 | explicit IRBuilder(BasicBlock *TheBB, const T &F, MDNode *FPMathTag = nullptr, | ||||||
881 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
882 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles), Folder(F) { | ||||||
883 | SetInsertPoint(TheBB); | ||||||
884 | } | ||||||
885 | |||||||
886 | explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr, | ||||||
887 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
888 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles) { | ||||||
889 | SetInsertPoint(TheBB); | ||||||
890 | } | ||||||
891 | |||||||
892 | explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr, | ||||||
893 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
894 | : IRBuilderBase(IP->getContext(), FPMathTag, OpBundles) { | ||||||
895 | SetInsertPoint(IP); | ||||||
896 | } | ||||||
897 | |||||||
898 | IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, const T &F, | ||||||
899 | MDNode *FPMathTag = nullptr, | ||||||
900 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
901 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles), Folder(F) { | ||||||
902 | SetInsertPoint(TheBB, IP); | ||||||
903 | } | ||||||
904 | |||||||
905 | IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, | ||||||
906 | MDNode *FPMathTag = nullptr, | ||||||
907 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
908 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles) { | ||||||
909 | SetInsertPoint(TheBB, IP); | ||||||
910 | } | ||||||
911 | |||||||
912 | /// Get the constant folder being used. | ||||||
913 | const T &getFolder() { return Folder; } | ||||||
914 | |||||||
915 | /// Insert and return the specified instruction. | ||||||
916 | template<typename InstTy> | ||||||
917 | InstTy *Insert(InstTy *I, const Twine &Name = "") const { | ||||||
918 | this->InsertHelper(I, Name, BB, InsertPt); | ||||||
919 | this->SetInstDebugLocation(I); | ||||||
920 | return I; | ||||||
921 | } | ||||||
922 | |||||||
923 | /// No-op overload to handle constants. | ||||||
924 | Constant *Insert(Constant *C, const Twine& = "") const { | ||||||
925 | return C; | ||||||
926 | } | ||||||
927 | |||||||
928 | //===--------------------------------------------------------------------===// | ||||||
929 | // Instruction creation methods: Terminators | ||||||
930 | //===--------------------------------------------------------------------===// | ||||||
931 | |||||||
932 | private: | ||||||
933 | /// Helper to add branch weight and unpredictable metadata onto an | ||||||
934 | /// instruction. | ||||||
935 | /// \returns The annotated instruction. | ||||||
936 | template <typename InstTy> | ||||||
937 | InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) { | ||||||
938 | if (Weights) | ||||||
939 | I->setMetadata(LLVMContext::MD_prof, Weights); | ||||||
940 | if (Unpredictable) | ||||||
941 | I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable); | ||||||
942 | return I; | ||||||
943 | } | ||||||
944 | |||||||
945 | public: | ||||||
946 | /// Create a 'ret void' instruction. | ||||||
947 | ReturnInst *CreateRetVoid() { | ||||||
948 | return Insert(ReturnInst::Create(Context)); | ||||||
949 | } | ||||||
950 | |||||||
951 | /// Create a 'ret <val>' instruction. | ||||||
952 | ReturnInst *CreateRet(Value *V) { | ||||||
953 | return Insert(ReturnInst::Create(Context, V)); | ||||||
954 | } | ||||||
955 | |||||||
956 | /// Create a sequence of N insertvalue instructions, | ||||||
957 | /// with one Value from the retVals array each, that build a aggregate | ||||||
958 | /// return value one value at a time, and a ret instruction to return | ||||||
959 | /// the resulting aggregate value. | ||||||
960 | /// | ||||||
961 | /// This is a convenience function for code that uses aggregate return values | ||||||
962 | /// as a vehicle for having multiple return values. | ||||||
963 | ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) { | ||||||
964 | Value *V = UndefValue::get(getCurrentFunctionReturnType()); | ||||||
965 | for (unsigned i = 0; i != N; ++i) | ||||||
966 | V = CreateInsertValue(V, retVals[i], i, "mrv"); | ||||||
967 | return Insert(ReturnInst::Create(Context, V)); | ||||||
968 | } | ||||||
969 | |||||||
970 | /// Create an unconditional 'br label X' instruction. | ||||||
971 | BranchInst *CreateBr(BasicBlock *Dest) { | ||||||
972 | return Insert(BranchInst::Create(Dest)); | ||||||
973 | } | ||||||
974 | |||||||
975 | /// Create a conditional 'br Cond, TrueDest, FalseDest' | ||||||
976 | /// instruction. | ||||||
977 | BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, | ||||||
978 | MDNode *BranchWeights = nullptr, | ||||||
979 | MDNode *Unpredictable = nullptr) { | ||||||
980 | return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond), | ||||||
981 | BranchWeights, Unpredictable)); | ||||||
982 | } | ||||||
983 | |||||||
984 | /// Create a conditional 'br Cond, TrueDest, FalseDest' | ||||||
985 | /// instruction. Copy branch meta data if available. | ||||||
986 | BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, | ||||||
987 | Instruction *MDSrc) { | ||||||
988 | BranchInst *Br = BranchInst::Create(True, False, Cond); | ||||||
989 | if (MDSrc) { | ||||||
990 | unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable, | ||||||
991 | LLVMContext::MD_make_implicit, LLVMContext::MD_dbg}; | ||||||
992 | Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4)); | ||||||
993 | } | ||||||
994 | return Insert(Br); | ||||||
995 | } | ||||||
996 | |||||||
997 | /// Create a switch instruction with the specified value, default dest, | ||||||
998 | /// and with a hint for the number of cases that will be added (for efficient | ||||||
999 | /// allocation). | ||||||
1000 | SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10, | ||||||
1001 | MDNode *BranchWeights = nullptr, | ||||||
1002 | MDNode *Unpredictable = nullptr) { | ||||||
1003 | return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases), | ||||||
1004 | BranchWeights, Unpredictable)); | ||||||
1005 | } | ||||||
1006 | |||||||
1007 | /// Create an indirect branch instruction with the specified address | ||||||
1008 | /// operand, with an optional hint for the number of destinations that will be | ||||||
1009 | /// added (for efficient allocation). | ||||||
1010 | IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) { | ||||||
1011 | return Insert(IndirectBrInst::Create(Addr, NumDests)); | ||||||
1012 | } | ||||||
1013 | |||||||
1014 | /// Create an invoke instruction. | ||||||
1015 | InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee, | ||||||
1016 | BasicBlock *NormalDest, BasicBlock *UnwindDest, | ||||||
1017 | ArrayRef<Value *> Args, | ||||||
1018 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1019 | const Twine &Name = "") { | ||||||
1020 | return Insert( | ||||||
1021 | InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles), | ||||||
1022 | Name); | ||||||
1023 | } | ||||||
1024 | InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee, | ||||||
1025 | BasicBlock *NormalDest, BasicBlock *UnwindDest, | ||||||
1026 | ArrayRef<Value *> Args = None, | ||||||
1027 | const Twine &Name = "") { | ||||||
1028 | return Insert(InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args), | ||||||
1029 | Name); | ||||||
1030 | } | ||||||
1031 | |||||||
1032 | InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest, | ||||||
1033 | BasicBlock *UnwindDest, ArrayRef<Value *> Args, | ||||||
1034 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1035 | const Twine &Name = "") { | ||||||
1036 | return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(), | ||||||
1037 | NormalDest, UnwindDest, Args, OpBundles, Name); | ||||||
1038 | } | ||||||
1039 | |||||||
1040 | InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest, | ||||||
1041 | BasicBlock *UnwindDest, | ||||||
1042 | ArrayRef<Value *> Args = None, | ||||||
1043 | const Twine &Name = "") { | ||||||
1044 | return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(), | ||||||
1045 | NormalDest, UnwindDest, Args, Name); | ||||||
1046 | } | ||||||
1047 | |||||||
1048 | // Deprecated [opaque pointer types] | ||||||
1049 | InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, | ||||||
1050 | BasicBlock *UnwindDest, ArrayRef<Value *> Args, | ||||||
1051 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1052 | const Twine &Name = "") { | ||||||
1053 | return CreateInvoke( | ||||||
1054 | cast<FunctionType>( | ||||||
1055 | cast<PointerType>(Callee->getType())->getElementType()), | ||||||
1056 | Callee, NormalDest, UnwindDest, Args, OpBundles, Name); | ||||||
1057 | } | ||||||
1058 | |||||||
1059 | // Deprecated [opaque pointer types] | ||||||
1060 | InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, | ||||||
1061 | BasicBlock *UnwindDest, | ||||||
1062 | ArrayRef<Value *> Args = None, | ||||||
1063 | const Twine &Name = "") { | ||||||
1064 | return CreateInvoke( | ||||||
1065 | cast<FunctionType>( | ||||||
1066 | cast<PointerType>(Callee->getType())->getElementType()), | ||||||
1067 | Callee, NormalDest, UnwindDest, Args, Name); | ||||||
1068 | } | ||||||
1069 | |||||||
1070 | /// \brief Create a callbr instruction. | ||||||
1071 | CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, | ||||||
1072 | BasicBlock *DefaultDest, | ||||||
1073 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1074 | ArrayRef<Value *> Args = None, | ||||||
1075 | const Twine &Name = "") { | ||||||
1076 | return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, | ||||||
1077 | Args), Name); | ||||||
1078 | } | ||||||
1079 | CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, | ||||||
1080 | BasicBlock *DefaultDest, | ||||||
1081 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1082 | ArrayRef<Value *> Args, | ||||||
1083 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1084 | const Twine &Name = "") { | ||||||
1085 | return Insert( | ||||||
1086 | CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, | ||||||
1087 | OpBundles), Name); | ||||||
1088 | } | ||||||
1089 | |||||||
1090 | CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, | ||||||
1091 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1092 | ArrayRef<Value *> Args = None, | ||||||
1093 | const Twine &Name = "") { | ||||||
1094 | return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), | ||||||
1095 | DefaultDest, IndirectDests, Args, Name); | ||||||
1096 | } | ||||||
1097 | CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, | ||||||
1098 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1099 | ArrayRef<Value *> Args, | ||||||
1100 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1101 | const Twine &Name = "") { | ||||||
1102 | return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), | ||||||
1103 | DefaultDest, IndirectDests, Args, Name); | ||||||
1104 | } | ||||||
1105 | |||||||
1106 | ResumeInst *CreateResume(Value *Exn) { | ||||||
1107 | return Insert(ResumeInst::Create(Exn)); | ||||||
1108 | } | ||||||
1109 | |||||||
1110 | CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad, | ||||||
1111 | BasicBlock *UnwindBB = nullptr) { | ||||||
1112 | return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB)); | ||||||
1113 | } | ||||||
1114 | |||||||
1115 | CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB, | ||||||
1116 | unsigned NumHandlers, | ||||||
1117 | const Twine &Name = "") { | ||||||
1118 | return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers), | ||||||
1119 | Name); | ||||||
1120 | } | ||||||
1121 | |||||||
1122 | CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args, | ||||||
1123 | const Twine &Name = "") { | ||||||
1124 | return Insert(CatchPadInst::Create(ParentPad, Args), Name); | ||||||
1125 | } | ||||||
1126 | |||||||
1127 | CleanupPadInst *CreateCleanupPad(Value *ParentPad, | ||||||
1128 | ArrayRef<Value *> Args = None, | ||||||
1129 | const Twine &Name = "") { | ||||||
1130 | return Insert(CleanupPadInst::Create(ParentPad, Args), Name); | ||||||
1131 | } | ||||||
1132 | |||||||
1133 | CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) { | ||||||
1134 | return Insert(CatchReturnInst::Create(CatchPad, BB)); | ||||||
1135 | } | ||||||
1136 | |||||||
1137 | UnreachableInst *CreateUnreachable() { | ||||||
1138 | return Insert(new UnreachableInst(Context)); | ||||||
1139 | } | ||||||
1140 | |||||||
1141 | //===--------------------------------------------------------------------===// | ||||||
1142 | // Instruction creation methods: Binary Operators | ||||||
1143 | //===--------------------------------------------------------------------===// | ||||||
1144 | private: | ||||||
1145 | BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc, | ||||||
1146 | Value *LHS, Value *RHS, | ||||||
1147 | const Twine &Name, | ||||||
1148 | bool HasNUW, bool HasNSW) { | ||||||
1149 | BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name); | ||||||
1150 | if (HasNUW) BO->setHasNoUnsignedWrap(); | ||||||
1151 | if (HasNSW) BO->setHasNoSignedWrap(); | ||||||
1152 | return BO; | ||||||
1153 | } | ||||||
1154 | |||||||
1155 | Instruction *setFPAttrs(Instruction *I, MDNode *FPMD, | ||||||
1156 | FastMathFlags FMF) const { | ||||||
1157 | if (!FPMD) | ||||||
1158 | FPMD = DefaultFPMathTag; | ||||||
1159 | if (FPMD) | ||||||
1160 | I->setMetadata(LLVMContext::MD_fpmath, FPMD); | ||||||
1161 | I->setFastMathFlags(FMF); | ||||||
1162 | return I; | ||||||
1163 | } | ||||||
1164 | |||||||
1165 | Value *foldConstant(Instruction::BinaryOps Opc, Value *L, | ||||||
1166 | Value *R, const Twine &Name) const { | ||||||
1167 | auto *LC = dyn_cast<Constant>(L); | ||||||
1168 | auto *RC = dyn_cast<Constant>(R); | ||||||
1169 | return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr; | ||||||
1170 | } | ||||||
1171 | |||||||
1172 | Value *getConstrainedFPRounding(Optional<fp::RoundingMode> Rounding) { | ||||||
1173 | fp::RoundingMode UseRounding = DefaultConstrainedRounding; | ||||||
1174 | |||||||
1175 | if (Rounding.hasValue()) | ||||||
1176 | UseRounding = Rounding.getValue(); | ||||||
1177 | |||||||
1178 | Optional<StringRef> RoundingStr = RoundingModeToStr(UseRounding); | ||||||
1179 | assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")((RoundingStr.hasValue() && "Garbage strict rounding mode!" ) ? static_cast<void> (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1179, __PRETTY_FUNCTION__)); | ||||||
1180 | auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue()); | ||||||
1181 | |||||||
1182 | return MetadataAsValue::get(Context, RoundingMDS); | ||||||
1183 | } | ||||||
1184 | |||||||
1185 | Value *getConstrainedFPExcept(Optional<fp::ExceptionBehavior> Except) { | ||||||
1186 | fp::ExceptionBehavior UseExcept = DefaultConstrainedExcept; | ||||||
1187 | |||||||
1188 | if (Except.hasValue()) | ||||||
1189 | UseExcept = Except.getValue(); | ||||||
1190 | |||||||
1191 | Optional<StringRef> ExceptStr = ExceptionBehaviorToStr(UseExcept); | ||||||
1192 | assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")((ExceptStr.hasValue() && "Garbage strict exception behavior!" ) ? static_cast<void> (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1192, __PRETTY_FUNCTION__)); | ||||||
1193 | auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue()); | ||||||
1194 | |||||||
1195 | return MetadataAsValue::get(Context, ExceptMDS); | ||||||
1196 | } | ||||||
1197 | |||||||
1198 | public: | ||||||
1199 | Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1200 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1201 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1202 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1203 | return Insert(Folder.CreateAdd(LC, RC, HasNUW, HasNSW), Name); | ||||||
1204 | return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, | ||||||
1205 | HasNUW, HasNSW); | ||||||
1206 | } | ||||||
1207 | |||||||
1208 | Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1209 | return CreateAdd(LHS, RHS, Name, false, true); | ||||||
1210 | } | ||||||
1211 | |||||||
1212 | Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1213 | return CreateAdd(LHS, RHS, Name, true, false); | ||||||
1214 | } | ||||||
1215 | |||||||
1216 | Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1217 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1218 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1219 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1220 | return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name); | ||||||
1221 | return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name, | ||||||
1222 | HasNUW, HasNSW); | ||||||
1223 | } | ||||||
1224 | |||||||
1225 | Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1226 | return CreateSub(LHS, RHS, Name, false, true); | ||||||
1227 | } | ||||||
1228 | |||||||
1229 | Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1230 | return CreateSub(LHS, RHS, Name, true, false); | ||||||
1231 | } | ||||||
1232 | |||||||
1233 | Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1234 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1235 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1236 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1237 | return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name); | ||||||
1238 | return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name, | ||||||
1239 | HasNUW, HasNSW); | ||||||
1240 | } | ||||||
1241 | |||||||
1242 | Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1243 | return CreateMul(LHS, RHS, Name, false, true); | ||||||
1244 | } | ||||||
1245 | |||||||
1246 | Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1247 | return CreateMul(LHS, RHS, Name, true, false); | ||||||
1248 | } | ||||||
1249 | |||||||
1250 | Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1251 | bool isExact = false) { | ||||||
1252 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1253 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1254 | return Insert(Folder.CreateUDiv(LC, RC, isExact), Name); | ||||||
1255 | if (!isExact) | ||||||
1256 | return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name); | ||||||
1257 | return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name); | ||||||
1258 | } | ||||||
1259 | |||||||
1260 | Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1261 | return CreateUDiv(LHS, RHS, Name, true); | ||||||
1262 | } | ||||||
1263 | |||||||
1264 | Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1265 | bool isExact = false) { | ||||||
1266 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1267 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1268 | return Insert(Folder.CreateSDiv(LC, RC, isExact), Name); | ||||||
1269 | if (!isExact) | ||||||
1270 | return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name); | ||||||
1271 | return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name); | ||||||
1272 | } | ||||||
1273 | |||||||
1274 | Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1275 | return CreateSDiv(LHS, RHS, Name, true); | ||||||
1276 | } | ||||||
1277 | |||||||
1278 | Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1279 | if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V; | ||||||
1280 | return Insert(BinaryOperator::CreateURem(LHS, RHS), Name); | ||||||
1281 | } | ||||||
1282 | |||||||
1283 | Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1284 | if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V; | ||||||
1285 | return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name); | ||||||
1286 | } | ||||||
1287 | |||||||
1288 | Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1289 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1290 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1291 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1292 | return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name); | ||||||
1293 | return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name, | ||||||
1294 | HasNUW, HasNSW); | ||||||
1295 | } | ||||||
1296 | |||||||
1297 | Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "", | ||||||
1298 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1299 | return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name, | ||||||
1300 | HasNUW, HasNSW); | ||||||
1301 | } | ||||||
1302 | |||||||
1303 | Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "", | ||||||
1304 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1305 | return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name, | ||||||
1306 | HasNUW, HasNSW); | ||||||
1307 | } | ||||||
1308 | |||||||
1309 | Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1310 | bool isExact = false) { | ||||||
1311 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1312 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1313 | return Insert(Folder.CreateLShr(LC, RC, isExact), Name); | ||||||
1314 | if (!isExact) | ||||||
1315 | return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name); | ||||||
1316 | return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name); | ||||||
1317 | } | ||||||
1318 | |||||||
1319 | Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "", | ||||||
1320 | bool isExact = false) { | ||||||
1321 | return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1322 | } | ||||||
1323 | |||||||
1324 | Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "", | ||||||
1325 | bool isExact = false) { | ||||||
1326 | return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1327 | } | ||||||
1328 | |||||||
1329 | Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1330 | bool isExact = false) { | ||||||
1331 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1332 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1333 | return Insert(Folder.CreateAShr(LC, RC, isExact), Name); | ||||||
1334 | if (!isExact) | ||||||
1335 | return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name); | ||||||
1336 | return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name); | ||||||
1337 | } | ||||||
1338 | |||||||
1339 | Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "", | ||||||
1340 | bool isExact = false) { | ||||||
1341 | return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1342 | } | ||||||
1343 | |||||||
1344 | Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "", | ||||||
1345 | bool isExact = false) { | ||||||
1346 | return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1347 | } | ||||||
1348 | |||||||
1349 | Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1350 | if (auto *RC = dyn_cast<Constant>(RHS)) { | ||||||
1351 | if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne()) | ||||||
1352 | return LHS; // LHS & -1 -> LHS | ||||||
1353 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1354 | return Insert(Folder.CreateAnd(LC, RC), Name); | ||||||
1355 | } | ||||||
1356 | return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name); | ||||||
1357 | } | ||||||
1358 | |||||||
1359 | Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") { | ||||||
1360 | return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1361 | } | ||||||
1362 | |||||||
1363 | Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") { | ||||||
1364 | return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1365 | } | ||||||
1366 | |||||||
1367 | Value *CreateAnd(ArrayRef<Value*> Ops) { | ||||||
1368 | assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail ("!Ops.empty()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1368, __PRETTY_FUNCTION__)); | ||||||
1369 | Value *Accum = Ops[0]; | ||||||
1370 | for (unsigned i = 1; i < Ops.size(); i++) | ||||||
1371 | Accum = CreateAnd(Accum, Ops[i]); | ||||||
1372 | return Accum; | ||||||
1373 | } | ||||||
1374 | |||||||
1375 | Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1376 | if (auto *RC = dyn_cast<Constant>(RHS)) { | ||||||
1377 | if (RC->isNullValue()) | ||||||
1378 | return LHS; // LHS | 0 -> LHS | ||||||
1379 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1380 | return Insert(Folder.CreateOr(LC, RC), Name); | ||||||
1381 | } | ||||||
1382 | return Insert(BinaryOperator::CreateOr(LHS, RHS), Name); | ||||||
1383 | } | ||||||
1384 | |||||||
1385 | Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") { | ||||||
1386 | return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1387 | } | ||||||
1388 | |||||||
1389 | Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") { | ||||||
1390 | return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1391 | } | ||||||
1392 | |||||||
1393 | Value *CreateOr(ArrayRef<Value*> Ops) { | ||||||
1394 | assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail ("!Ops.empty()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1394, __PRETTY_FUNCTION__)); | ||||||
1395 | Value *Accum = Ops[0]; | ||||||
1396 | for (unsigned i = 1; i < Ops.size(); i++) | ||||||
1397 | Accum = CreateOr(Accum, Ops[i]); | ||||||
1398 | return Accum; | ||||||
1399 | } | ||||||
1400 | |||||||
1401 | Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1402 | if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V; | ||||||
1403 | return Insert(BinaryOperator::CreateXor(LHS, RHS), Name); | ||||||
1404 | } | ||||||
1405 | |||||||
1406 | Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") { | ||||||
1407 | return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1408 | } | ||||||
1409 | |||||||
1410 | Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") { | ||||||
1411 | return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1412 | } | ||||||
1413 | |||||||
1414 | Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "", | ||||||
1415 | MDNode *FPMD = nullptr) { | ||||||
1416 | if (IsFPConstrained) | ||||||
1417 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, | ||||||
1418 | L, R, nullptr, Name, FPMD); | ||||||
1419 | |||||||
1420 | if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; | ||||||
1421 | Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF); | ||||||
1422 | return Insert(I, Name); | ||||||
1423 | } | ||||||
1424 | |||||||
1425 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1426 | /// default FMF. | ||||||
1427 | Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1428 | const Twine &Name = "") { | ||||||
1429 | if (IsFPConstrained) | ||||||
1430 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, | ||||||
1431 | L, R, FMFSource, Name); | ||||||
1432 | |||||||
1433 | if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; | ||||||
1434 | Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, | ||||||
1435 | FMFSource->getFastMathFlags()); | ||||||
1436 | return Insert(I, Name); | ||||||
1437 | } | ||||||
1438 | |||||||
1439 | Value *CreateFSub(Value *L, Value *R, const Twine &Name = "", | ||||||
1440 | MDNode *FPMD = nullptr) { | ||||||
1441 | if (IsFPConstrained) | ||||||
1442 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, | ||||||
1443 | L, R, nullptr, Name, FPMD); | ||||||
1444 | |||||||
1445 | if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; | ||||||
1446 | Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF); | ||||||
1447 | return Insert(I, Name); | ||||||
1448 | } | ||||||
1449 | |||||||
1450 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1451 | /// default FMF. | ||||||
1452 | Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1453 | const Twine &Name = "") { | ||||||
1454 | if (IsFPConstrained) | ||||||
1455 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, | ||||||
1456 | L, R, FMFSource, Name); | ||||||
1457 | |||||||
1458 | if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; | ||||||
1459 | Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, | ||||||
1460 | FMFSource->getFastMathFlags()); | ||||||
1461 | return Insert(I, Name); | ||||||
1462 | } | ||||||
1463 | |||||||
1464 | Value *CreateFMul(Value *L, Value *R, const Twine &Name = "", | ||||||
1465 | MDNode *FPMD = nullptr) { | ||||||
1466 | if (IsFPConstrained) | ||||||
1467 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, | ||||||
1468 | L, R, nullptr, Name, FPMD); | ||||||
1469 | |||||||
1470 | if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; | ||||||
1471 | Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF); | ||||||
1472 | return Insert(I, Name); | ||||||
1473 | } | ||||||
1474 | |||||||
1475 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1476 | /// default FMF. | ||||||
1477 | Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1478 | const Twine &Name = "") { | ||||||
1479 | if (IsFPConstrained) | ||||||
1480 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, | ||||||
1481 | L, R, FMFSource, Name); | ||||||
1482 | |||||||
1483 | if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; | ||||||
1484 | Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, | ||||||
1485 | FMFSource->getFastMathFlags()); | ||||||
1486 | return Insert(I, Name); | ||||||
1487 | } | ||||||
1488 | |||||||
1489 | Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "", | ||||||
1490 | MDNode *FPMD = nullptr) { | ||||||
1491 | if (IsFPConstrained) | ||||||
1492 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, | ||||||
1493 | L, R, nullptr, Name, FPMD); | ||||||
1494 | |||||||
1495 | if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; | ||||||
1496 | Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF); | ||||||
1497 | return Insert(I, Name); | ||||||
1498 | } | ||||||
1499 | |||||||
1500 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1501 | /// default FMF. | ||||||
1502 | Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1503 | const Twine &Name = "") { | ||||||
1504 | if (IsFPConstrained) | ||||||
1505 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, | ||||||
1506 | L, R, FMFSource, Name); | ||||||
1507 | |||||||
1508 | if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; | ||||||
1509 | Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, | ||||||
1510 | FMFSource->getFastMathFlags()); | ||||||
1511 | return Insert(I, Name); | ||||||
1512 | } | ||||||
1513 | |||||||
1514 | Value *CreateFRem(Value *L, Value *R, const Twine &Name = "", | ||||||
1515 | MDNode *FPMD = nullptr) { | ||||||
1516 | if (IsFPConstrained) | ||||||
1517 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, | ||||||
1518 | L, R, nullptr, Name, FPMD); | ||||||
1519 | |||||||
1520 | if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; | ||||||
1521 | Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF); | ||||||
1522 | return Insert(I, Name); | ||||||
1523 | } | ||||||
1524 | |||||||
1525 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1526 | /// default FMF. | ||||||
1527 | Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1528 | const Twine &Name = "") { | ||||||
1529 | if (IsFPConstrained) | ||||||
1530 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, | ||||||
1531 | L, R, FMFSource, Name); | ||||||
1532 | |||||||
1533 | if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; | ||||||
1534 | Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, | ||||||
1535 | FMFSource->getFastMathFlags()); | ||||||
1536 | return Insert(I, Name); | ||||||
1537 | } | ||||||
1538 | |||||||
1539 | Value *CreateBinOp(Instruction::BinaryOps Opc, | ||||||
1540 | Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1541 | MDNode *FPMathTag = nullptr) { | ||||||
1542 | if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V; | ||||||
1543 | Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS); | ||||||
1544 | if (isa<FPMathOperator>(BinOp)) | ||||||
1545 | setFPAttrs(BinOp, FPMathTag, FMF); | ||||||
1546 | return Insert(BinOp, Name); | ||||||
1547 | } | ||||||
1548 | |||||||
1549 | CallInst *CreateConstrainedFPBinOp( | ||||||
1550 | Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr, | ||||||
1551 | const Twine &Name = "", MDNode *FPMathTag = nullptr, | ||||||
1552 | Optional<fp::RoundingMode> Rounding = None, | ||||||
1553 | Optional<fp::ExceptionBehavior> Except = None) { | ||||||
1554 | Value *RoundingV = getConstrainedFPRounding(Rounding); | ||||||
1555 | Value *ExceptV = getConstrainedFPExcept(Except); | ||||||
1556 | |||||||
1557 | FastMathFlags UseFMF = FMF; | ||||||
1558 | if (FMFSource) | ||||||
1559 | UseFMF = FMFSource->getFastMathFlags(); | ||||||
1560 | |||||||
1561 | CallInst *C = CreateIntrinsic(ID, {L->getType()}, | ||||||
1562 | {L, R, RoundingV, ExceptV}, nullptr, Name); | ||||||
1563 | setConstrainedFPCallAttr(C); | ||||||
1564 | setFPAttrs(C, FPMathTag, UseFMF); | ||||||
1565 | return C; | ||||||
1566 | } | ||||||
1567 | |||||||
1568 | Value *CreateNeg(Value *V, const Twine &Name = "", | ||||||
1569 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1570 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1571 | return Insert(Folder.CreateNeg(VC, HasNUW, HasNSW), Name); | ||||||
1572 | BinaryOperator *BO = Insert(BinaryOperator::CreateNeg(V), Name); | ||||||
1573 | if (HasNUW) BO->setHasNoUnsignedWrap(); | ||||||
1574 | if (HasNSW) BO->setHasNoSignedWrap(); | ||||||
1575 | return BO; | ||||||
1576 | } | ||||||
1577 | |||||||
1578 | Value *CreateNSWNeg(Value *V, const Twine &Name = "") { | ||||||
1579 | return CreateNeg(V, Name, false, true); | ||||||
1580 | } | ||||||
1581 | |||||||
1582 | Value *CreateNUWNeg(Value *V, const Twine &Name = "") { | ||||||
1583 | return CreateNeg(V, Name, true, false); | ||||||
1584 | } | ||||||
1585 | |||||||
1586 | Value *CreateFNeg(Value *V, const Twine &Name = "", | ||||||
1587 | MDNode *FPMathTag = nullptr) { | ||||||
1588 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1589 | return Insert(Folder.CreateFNeg(VC), Name); | ||||||
1590 | return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF), | ||||||
1591 | Name); | ||||||
1592 | } | ||||||
1593 | |||||||
1594 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1595 | /// default FMF. | ||||||
1596 | Value *CreateFNegFMF(Value *V, Instruction *FMFSource, | ||||||
1597 | const Twine &Name = "") { | ||||||
1598 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1599 | return Insert(Folder.CreateFNeg(VC), Name); | ||||||
1600 | return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr, | ||||||
1601 | FMFSource->getFastMathFlags()), | ||||||
1602 | Name); | ||||||
1603 | } | ||||||
1604 | |||||||
1605 | Value *CreateNot(Value *V, const Twine &Name = "") { | ||||||
1606 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1607 | return Insert(Folder.CreateNot(VC), Name); | ||||||
1608 | return Insert(BinaryOperator::CreateNot(V), Name); | ||||||
1609 | } | ||||||
1610 | |||||||
1611 | Value *CreateUnOp(Instruction::UnaryOps Opc, | ||||||
1612 | Value *V, const Twine &Name = "", | ||||||
1613 | MDNode *FPMathTag = nullptr) { | ||||||
1614 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1615 | return Insert(Folder.CreateUnOp(Opc, VC), Name); | ||||||
1616 | Instruction *UnOp = UnaryOperator::Create(Opc, V); | ||||||
1617 | if (isa<FPMathOperator>(UnOp)) | ||||||
1618 | setFPAttrs(UnOp, FPMathTag, FMF); | ||||||
1619 | return Insert(UnOp, Name); | ||||||
1620 | } | ||||||
1621 | |||||||
1622 | /// Create either a UnaryOperator or BinaryOperator depending on \p Opc. | ||||||
1623 | /// Correct number of operands must be passed accordingly. | ||||||
1624 | Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, | ||||||
1625 | const Twine &Name = "", | ||||||
1626 | MDNode *FPMathTag = nullptr) { | ||||||
1627 | if (Instruction::isBinaryOp(Opc)) { | ||||||
1628 | assert(Ops.size() == 2 && "Invalid number of operands!")((Ops.size() == 2 && "Invalid number of operands!") ? static_cast<void> (0) : __assert_fail ("Ops.size() == 2 && \"Invalid number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1628, __PRETTY_FUNCTION__)); | ||||||
1629 | return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc), | ||||||
1630 | Ops[0], Ops[1], Name, FPMathTag); | ||||||
1631 | } | ||||||
1632 | if (Instruction::isUnaryOp(Opc)) { | ||||||
1633 | assert(Ops.size() == 1 && "Invalid number of operands!")((Ops.size() == 1 && "Invalid number of operands!") ? static_cast<void> (0) : __assert_fail ("Ops.size() == 1 && \"Invalid number of operands!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1633, __PRETTY_FUNCTION__)); | ||||||
1634 | return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc), | ||||||
1635 | Ops[0], Name, FPMathTag); | ||||||
1636 | } | ||||||
1637 | llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 1637); | ||||||
1638 | } | ||||||
1639 | |||||||
1640 | //===--------------------------------------------------------------------===// | ||||||
1641 | // Instruction creation methods: Memory Instructions | ||||||
1642 | //===--------------------------------------------------------------------===// | ||||||
1643 | |||||||
1644 | AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace, | ||||||
1645 | Value *ArraySize = nullptr, const Twine &Name = "") { | ||||||
1646 | return Insert(new AllocaInst(Ty, AddrSpace, ArraySize), Name); | ||||||
1647 | } | ||||||
1648 | |||||||
1649 | AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr, | ||||||
1650 | const Twine &Name = "") { | ||||||
1651 | const DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); | ||||||
1652 | return Insert(new AllocaInst(Ty, DL.getAllocaAddrSpace(), ArraySize), Name); | ||||||
1653 | } | ||||||
1654 | |||||||
1655 | /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of | ||||||
1656 | /// converting the string to 'bool' for the isVolatile parameter. | ||||||
1657 | LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) { | ||||||
1658 | return Insert(new LoadInst(Ty, Ptr), Name); | ||||||
1659 | } | ||||||
1660 | |||||||
1661 | LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") { | ||||||
1662 | return Insert(new LoadInst(Ty, Ptr), Name); | ||||||
1663 | } | ||||||
1664 | |||||||
1665 | LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile, | ||||||
1666 | const Twine &Name = "") { | ||||||
1667 | return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile), Name); | ||||||
1668 | } | ||||||
1669 | |||||||
1670 | // Deprecated [opaque pointer types] | ||||||
1671 | LoadInst *CreateLoad(Value *Ptr, const char *Name) { | ||||||
1672 | return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name); | ||||||
1673 | } | ||||||
1674 | |||||||
1675 | // Deprecated [opaque pointer types] | ||||||
1676 | LoadInst *CreateLoad(Value *Ptr, const Twine &Name = "") { | ||||||
1677 | return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name); | ||||||
1678 | } | ||||||
1679 | |||||||
1680 | // Deprecated [opaque pointer types] | ||||||
1681 | LoadInst *CreateLoad(Value *Ptr, bool isVolatile, const Twine &Name = "") { | ||||||
1682 | return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, isVolatile, | ||||||
1683 | Name); | ||||||
1684 | } | ||||||
1685 | |||||||
1686 | StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) { | ||||||
1687 | return Insert(new StoreInst(Val, Ptr, isVolatile)); | ||||||
1688 | } | ||||||
1689 | |||||||
1690 | /// Provided to resolve 'CreateAlignedLoad(Ptr, Align, "...")' | ||||||
1691 | /// correctly, instead of converting the string to 'bool' for the isVolatile | ||||||
1692 | /// parameter. | ||||||
1693 | /// FIXME: Remove this function once transition to Align is over. | ||||||
1694 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
1695 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, | ||||||
1696 | const char *Name) { | ||||||
1697 | return CreateAlignedLoad(Ty, Ptr, MaybeAlign(Align), Name); | ||||||
1698 | } | ||||||
1699 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, | ||||||
1700 | const char *Name) { | ||||||
1701 | LoadInst *LI = CreateLoad(Ty, Ptr, Name); | ||||||
1702 | LI->setAlignment(Align); | ||||||
1703 | return LI; | ||||||
1704 | } | ||||||
1705 | /// FIXME: Remove this function once transition to Align is over. | ||||||
1706 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
1707 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, | ||||||
1708 | const Twine &Name = "") { | ||||||
1709 | return CreateAlignedLoad(Ty, Ptr, MaybeAlign(Align), Name); | ||||||
1710 | } | ||||||
1711 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, | ||||||
1712 | const Twine &Name = "") { | ||||||
1713 | LoadInst *LI = CreateLoad(Ty, Ptr, Name); | ||||||
1714 | LI->setAlignment(Align); | ||||||
1715 | return LI; | ||||||
1716 | } | ||||||
1717 | /// FIXME: Remove this function once transition to Align is over. | ||||||
1718 | /// Use the version that takes MaybeAlign instead of this one. | ||||||
1719 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, | ||||||
1720 | bool isVolatile, const Twine &Name = "") { | ||||||
1721 | return CreateAlignedLoad(Ty, Ptr, MaybeAlign(Align), isVolatile, Name); | ||||||
1722 | } | ||||||
1723 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, | ||||||
1724 | bool isVolatile, const Twine &Name = "") { | ||||||
1725 | LoadInst *LI = CreateLoad(Ty, Ptr, isVolatile, Name); | ||||||
1726 | LI->setAlignment(Align); | ||||||
1727 | return LI; | ||||||
1728 | } | ||||||
1729 | |||||||
1730 | // Deprecated [opaque pointer types] | ||||||
1731 | LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, const char *Name) { | ||||||
1732 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1733 | Align, Name); | ||||||
1734 | } | ||||||
1735 | // Deprecated [opaque pointer types] | ||||||
1736 | LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, | ||||||
1737 | const Twine &Name = "") { | ||||||
1738 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1739 | Align, Name); | ||||||
1740 | } | ||||||
1741 | // Deprecated [opaque pointer types] | ||||||
1742 | LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, bool isVolatile, | ||||||
1743 | const Twine &Name = "") { | ||||||
1744 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1745 | Align, isVolatile, Name); | ||||||
1746 | } | ||||||
1747 | // Deprecated [opaque pointer types] | ||||||
1748 | LoadInst *CreateAlignedLoad(Value *Ptr, MaybeAlign Align, const char *Name) { | ||||||
1749 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1750 | Align, Name); | ||||||
1751 | } | ||||||
1752 | // Deprecated [opaque pointer types] | ||||||
1753 | LoadInst *CreateAlignedLoad(Value *Ptr, MaybeAlign Align, | ||||||
1754 | const Twine &Name = "") { | ||||||
1755 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1756 | Align, Name); | ||||||
1757 | } | ||||||
1758 | // Deprecated [opaque pointer types] | ||||||
1759 | LoadInst *CreateAlignedLoad(Value *Ptr, MaybeAlign Align, bool isVolatile, | ||||||
1760 | const Twine &Name = "") { | ||||||
1761 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1762 | Align, isVolatile, Name); | ||||||
1763 | } | ||||||
1764 | |||||||
1765 | StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align, | ||||||
1766 | bool isVolatile = false) { | ||||||
1767 | StoreInst *SI = CreateStore(Val, Ptr, isVolatile); | ||||||
1768 | SI->setAlignment(MaybeAlign(Align)); | ||||||
1769 | return SI; | ||||||
1770 | } | ||||||
1771 | StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, | ||||||
1772 | bool isVolatile = false) { | ||||||
1773 | return CreateAlignedStore(Val, Ptr, Align ? Align->value() : 0, isVolatile); | ||||||
1774 | } | ||||||
1775 | FenceInst *CreateFence(AtomicOrdering Ordering, | ||||||
1776 | SyncScope::ID SSID = SyncScope::System, | ||||||
1777 | const Twine &Name = "") { | ||||||
1778 | return Insert(new FenceInst(Context, Ordering, SSID), Name); | ||||||
1779 | } | ||||||
1780 | |||||||
1781 | AtomicCmpXchgInst * | ||||||
1782 | CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, | ||||||
1783 | AtomicOrdering SuccessOrdering, | ||||||
1784 | AtomicOrdering FailureOrdering, | ||||||
1785 | SyncScope::ID SSID = SyncScope::System) { | ||||||
1786 | return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, | ||||||
1787 | FailureOrdering, SSID)); | ||||||
1788 | } | ||||||
1789 | |||||||
1790 | AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, | ||||||
1791 | AtomicOrdering Ordering, | ||||||
1792 | SyncScope::ID SSID = SyncScope::System) { | ||||||
1793 | return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SSID)); | ||||||
1794 | } | ||||||
1795 | |||||||
1796 | Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1797 | const Twine &Name = "") { | ||||||
1798 | return CreateGEP(nullptr, Ptr, IdxList, Name); | ||||||
1799 | } | ||||||
1800 | |||||||
1801 | Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1802 | const Twine &Name = "") { | ||||||
1803 | if (auto *PC = dyn_cast<Constant>(Ptr)) { | ||||||
1804 | // Every index must be constant. | ||||||
1805 | size_t i, e; | ||||||
1806 | for (i = 0, e = IdxList.size(); i != e; ++i) | ||||||
1807 | if (!isa<Constant>(IdxList[i])) | ||||||
1808 | break; | ||||||
1809 | if (i == e) | ||||||
1810 | return Insert(Folder.CreateGetElementPtr(Ty, PC, IdxList), Name); | ||||||
1811 | } | ||||||
1812 | return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name); | ||||||
1813 | } | ||||||
1814 | |||||||
1815 | Value *CreateInBoundsGEP(Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1816 | const Twine &Name = "") { | ||||||
1817 | return CreateInBoundsGEP(nullptr, Ptr, IdxList, Name); | ||||||
1818 | } | ||||||
1819 | |||||||
1820 | Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1821 | const Twine &Name = "") { | ||||||
1822 | if (auto *PC = dyn_cast<Constant>(Ptr)) { | ||||||
1823 | // Every index must be constant. | ||||||
1824 | size_t i, e; | ||||||
1825 | for (i = 0, e = IdxList.size(); i != e; ++i) | ||||||
1826 | if (!isa<Constant>(IdxList[i])) | ||||||
1827 | break; | ||||||
1828 | if (i == e) | ||||||
1829 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IdxList), | ||||||
1830 | Name); | ||||||
1831 | } | ||||||
1832 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name); | ||||||
1833 | } | ||||||
1834 | |||||||
1835 | Value *CreateGEP(Value *Ptr, Value *Idx, const Twine &Name = "") { | ||||||
1836 | return CreateGEP(nullptr, Ptr, Idx, Name); | ||||||
1837 | } | ||||||
1838 | |||||||
1839 | Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") { | ||||||
1840 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1841 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
1842 | return Insert(Folder.CreateGetElementPtr(Ty, PC, IC), Name); | ||||||
1843 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); | ||||||
1844 | } | ||||||
1845 | |||||||
1846 | Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx, | ||||||
1847 | const Twine &Name = "") { | ||||||
1848 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1849 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
1850 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IC), Name); | ||||||
1851 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); | ||||||
1852 | } | ||||||
1853 | |||||||
1854 | Value *CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name = "") { | ||||||
1855 | return CreateConstGEP1_32(nullptr, Ptr, Idx0, Name); | ||||||
1856 | } | ||||||
1857 | |||||||
1858 | Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, | ||||||
1859 | const Twine &Name = "") { | ||||||
1860 | Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0); | ||||||
1861 | |||||||
1862 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1863 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name); | ||||||
1864 | |||||||
1865 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); | ||||||
1866 | } | ||||||
1867 | |||||||
1868 | Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, | ||||||
1869 | const Twine &Name = "") { | ||||||
1870 | Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0); | ||||||
1871 | |||||||
1872 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1873 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name); | ||||||
1874 | |||||||
1875 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); | ||||||
1876 | } | ||||||
1877 | |||||||
1878 | Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, | ||||||
1879 | const Twine &Name = "") { | ||||||
1880 | Value *Idxs[] = { | ||||||
1881 | ConstantInt::get(Type::getInt32Ty(Context), Idx0), | ||||||
1882 | ConstantInt::get(Type::getInt32Ty(Context), Idx1) | ||||||
1883 | }; | ||||||
1884 | |||||||
1885 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1886 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1887 | |||||||
1888 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name); | ||||||
1889 | } | ||||||
1890 | |||||||
1891 | Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, | ||||||
1892 | unsigned Idx1, const Twine &Name = "") { | ||||||
1893 | Value *Idxs[] = { | ||||||
1894 | ConstantInt::get(Type::getInt32Ty(Context), Idx0), | ||||||
1895 | ConstantInt::get(Type::getInt32Ty(Context), Idx1) | ||||||
1896 | }; | ||||||
1897 | |||||||
1898 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1899 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1900 | |||||||
1901 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name); | ||||||
1902 | } | ||||||
1903 | |||||||
1904 | Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0, | ||||||
1905 | const Twine &Name = "") { | ||||||
1906 | Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0); | ||||||
1907 | |||||||
1908 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1909 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name); | ||||||
1910 | |||||||
1911 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); | ||||||
1912 | } | ||||||
1913 | |||||||
1914 | Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") { | ||||||
1915 | return CreateConstGEP1_64(nullptr, Ptr, Idx0, Name); | ||||||
1916 | } | ||||||
1917 | |||||||
1918 | Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0, | ||||||
1919 | const Twine &Name = "") { | ||||||
1920 | Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0); | ||||||
1921 | |||||||
1922 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1923 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name); | ||||||
1924 | |||||||
1925 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); | ||||||
1926 | } | ||||||
1927 | |||||||
1928 | Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0, | ||||||
1929 | const Twine &Name = "") { | ||||||
1930 | return CreateConstInBoundsGEP1_64(nullptr, Ptr, Idx0, Name); | ||||||
1931 | } | ||||||
1932 | |||||||
1933 | Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, | ||||||
1934 | const Twine &Name = "") { | ||||||
1935 | Value *Idxs[] = { | ||||||
1936 | ConstantInt::get(Type::getInt64Ty(Context), Idx0), | ||||||
1937 | ConstantInt::get(Type::getInt64Ty(Context), Idx1) | ||||||
1938 | }; | ||||||
1939 | |||||||
1940 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1941 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1942 | |||||||
1943 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name); | ||||||
1944 | } | ||||||
1945 | |||||||
1946 | Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1, | ||||||
1947 | const Twine &Name = "") { | ||||||
1948 | return CreateConstGEP2_64(nullptr, Ptr, Idx0, Idx1, Name); | ||||||
1949 | } | ||||||
1950 | |||||||
1951 | Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, | ||||||
1952 | uint64_t Idx1, const Twine &Name = "") { | ||||||
1953 | Value *Idxs[] = { | ||||||
1954 | ConstantInt::get(Type::getInt64Ty(Context), Idx0), | ||||||
1955 | ConstantInt::get(Type::getInt64Ty(Context), Idx1) | ||||||
1956 | }; | ||||||
1957 | |||||||
1958 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1959 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1960 | |||||||
1961 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name); | ||||||
1962 | } | ||||||
1963 | |||||||
1964 | Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1, | ||||||
1965 | const Twine &Name = "") { | ||||||
1966 | return CreateConstInBoundsGEP2_64(nullptr, Ptr, Idx0, Idx1, Name); | ||||||
1967 | } | ||||||
1968 | |||||||
1969 | Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, | ||||||
1970 | const Twine &Name = "") { | ||||||
1971 | return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name); | ||||||
1972 | } | ||||||
1973 | |||||||
1974 | Value *CreateStructGEP(Value *Ptr, unsigned Idx, const Twine &Name = "") { | ||||||
1975 | return CreateConstInBoundsGEP2_32(nullptr, Ptr, 0, Idx, Name); | ||||||
1976 | } | ||||||
1977 | |||||||
1978 | /// Same as CreateGlobalString, but return a pointer with "i8*" type | ||||||
1979 | /// instead of a pointer to array of i8. | ||||||
1980 | Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "", | ||||||
1981 | unsigned AddressSpace = 0) { | ||||||
1982 | GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace); | ||||||
1983 | Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); | ||||||
1984 | Constant *Indices[] = {Zero, Zero}; | ||||||
1985 | return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, | ||||||
1986 | Indices); | ||||||
1987 | } | ||||||
1988 | |||||||
1989 | //===--------------------------------------------------------------------===// | ||||||
1990 | // Instruction creation methods: Cast/Conversion Operators | ||||||
1991 | //===--------------------------------------------------------------------===// | ||||||
1992 | |||||||
1993 | Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1994 | return CreateCast(Instruction::Trunc, V, DestTy, Name); | ||||||
1995 | } | ||||||
1996 | |||||||
1997 | Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1998 | return CreateCast(Instruction::ZExt, V, DestTy, Name); | ||||||
1999 | } | ||||||
2000 | |||||||
2001 | Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
2002 | return CreateCast(Instruction::SExt, V, DestTy, Name); | ||||||
2003 | } | ||||||
2004 | |||||||
2005 | /// Create a ZExt or Trunc from the integer value V to DestTy. Return | ||||||
2006 | /// the value untouched if the type of V is already DestTy. | ||||||
2007 | Value *CreateZExtOrTrunc(Value *V, Type *DestTy, | ||||||
2008 | const Twine &Name = "") { | ||||||
2009 | assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only zero extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2011, __PRETTY_FUNCTION__)) | ||||||
2010 | DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only zero extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2011, __PRETTY_FUNCTION__)) | ||||||
2011 | "Can only zero extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only zero extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2011, __PRETTY_FUNCTION__)); | ||||||
2012 | Type *VTy = V->getType(); | ||||||
2013 | if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits()) | ||||||
2014 | return CreateZExt(V, DestTy, Name); | ||||||
2015 | if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits()) | ||||||
2016 | return CreateTrunc(V, DestTy, Name); | ||||||
2017 | return V; | ||||||
2018 | } | ||||||
2019 | |||||||
2020 | /// Create a SExt or Trunc from the integer value V to DestTy. Return | ||||||
2021 | /// the value untouched if the type of V is already DestTy. | ||||||
2022 | Value *CreateSExtOrTrunc(Value *V, Type *DestTy, | ||||||
2023 | const Twine &Name = "") { | ||||||
2024 | assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only sign extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2026, __PRETTY_FUNCTION__)) | ||||||
2025 | DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only sign extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2026, __PRETTY_FUNCTION__)) | ||||||
2026 | "Can only sign extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only sign extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2026, __PRETTY_FUNCTION__)); | ||||||
2027 | Type *VTy = V->getType(); | ||||||
2028 | if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits()) | ||||||
2029 | return CreateSExt(V, DestTy, Name); | ||||||
2030 | if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits()) | ||||||
2031 | return CreateTrunc(V, DestTy, Name); | ||||||
2032 | return V; | ||||||
2033 | } | ||||||
2034 | |||||||
2035 | Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
2036 | if (IsFPConstrained) | ||||||
2037 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui, | ||||||
2038 | V, DestTy, nullptr, Name); | ||||||
2039 | return CreateCast(Instruction::FPToUI, V, DestTy, Name); | ||||||
2040 | } | ||||||
2041 | |||||||
2042 | Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
2043 | if (IsFPConstrained) | ||||||
2044 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi, | ||||||
2045 | V, DestTy, nullptr, Name); | ||||||
2046 | return CreateCast(Instruction::FPToSI, V, DestTy, Name); | ||||||
2047 | } | ||||||
2048 | |||||||
2049 | Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){ | ||||||
2050 | if (IsFPConstrained) | ||||||
2051 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_uitofp, | ||||||
2052 | V, DestTy, nullptr, Name); | ||||||
2053 | return CreateCast(Instruction::UIToFP, V, DestTy, Name); | ||||||
2054 | } | ||||||
2055 | |||||||
2056 | Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){ | ||||||
2057 | if (IsFPConstrained) | ||||||
2058 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_sitofp, | ||||||
2059 | V, DestTy, nullptr, Name); | ||||||
2060 | return CreateCast(Instruction::SIToFP, V, DestTy, Name); | ||||||
2061 | } | ||||||
2062 | |||||||
2063 | Value *CreateFPTrunc(Value *V, Type *DestTy, | ||||||
2064 | const Twine &Name = "") { | ||||||
2065 | if (IsFPConstrained) | ||||||
2066 | return CreateConstrainedFPCast( | ||||||
2067 | Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr, | ||||||
2068 | Name); | ||||||
2069 | return CreateCast(Instruction::FPTrunc, V, DestTy, Name); | ||||||
2070 | } | ||||||
2071 | |||||||
2072 | Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
2073 | if (IsFPConstrained) | ||||||
2074 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext, | ||||||
2075 | V, DestTy, nullptr, Name); | ||||||
2076 | return CreateCast(Instruction::FPExt, V, DestTy, Name); | ||||||
2077 | } | ||||||
2078 | |||||||
2079 | Value *CreatePtrToInt(Value *V, Type *DestTy, | ||||||
2080 | const Twine &Name = "") { | ||||||
2081 | return CreateCast(Instruction::PtrToInt, V, DestTy, Name); | ||||||
2082 | } | ||||||
2083 | |||||||
2084 | Value *CreateIntToPtr(Value *V, Type *DestTy, | ||||||
2085 | const Twine &Name = "") { | ||||||
2086 | return CreateCast(Instruction::IntToPtr, V, DestTy, Name); | ||||||
2087 | } | ||||||
2088 | |||||||
2089 | Value *CreateBitCast(Value *V, Type *DestTy, | ||||||
2090 | const Twine &Name = "") { | ||||||
2091 | return CreateCast(Instruction::BitCast, V, DestTy, Name); | ||||||
2092 | } | ||||||
2093 | |||||||
2094 | Value *CreateAddrSpaceCast(Value *V, Type *DestTy, | ||||||
2095 | const Twine &Name = "") { | ||||||
2096 | return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name); | ||||||
2097 | } | ||||||
2098 | |||||||
2099 | Value *CreateZExtOrBitCast(Value *V, Type *DestTy, | ||||||
2100 | const Twine &Name = "") { | ||||||
2101 | if (V->getType() == DestTy) | ||||||
2102 | return V; | ||||||
2103 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2104 | return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name); | ||||||
2105 | return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name); | ||||||
2106 | } | ||||||
2107 | |||||||
2108 | Value *CreateSExtOrBitCast(Value *V, Type *DestTy, | ||||||
2109 | const Twine &Name = "") { | ||||||
2110 | if (V->getType() == DestTy) | ||||||
2111 | return V; | ||||||
2112 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2113 | return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name); | ||||||
2114 | return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name); | ||||||
2115 | } | ||||||
2116 | |||||||
2117 | Value *CreateTruncOrBitCast(Value *V, Type *DestTy, | ||||||
2118 | const Twine &Name = "") { | ||||||
2119 | if (V->getType() == DestTy) | ||||||
2120 | return V; | ||||||
2121 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2122 | return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name); | ||||||
2123 | return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name); | ||||||
2124 | } | ||||||
2125 | |||||||
2126 | Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, | ||||||
2127 | const Twine &Name = "") { | ||||||
2128 | if (V->getType() == DestTy) | ||||||
2129 | return V; | ||||||
2130 | if (auto *VC
| ||||||
2131 | return Insert(Folder.CreateCast(Op, VC, DestTy), Name); | ||||||
2132 | return Insert(CastInst::Create(Op, V, DestTy), Name); | ||||||
2133 | } | ||||||
2134 | |||||||
2135 | Value *CreatePointerCast(Value *V, Type *DestTy, | ||||||
2136 | const Twine &Name = "") { | ||||||
2137 | if (V->getType() == DestTy) | ||||||
2138 | return V; | ||||||
2139 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2140 | return Insert(Folder.CreatePointerCast(VC, DestTy), Name); | ||||||
2141 | return Insert(CastInst::CreatePointerCast(V, DestTy), Name); | ||||||
2142 | } | ||||||
2143 | |||||||
2144 | Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy, | ||||||
2145 | const Twine &Name = "") { | ||||||
2146 | if (V->getType() == DestTy) | ||||||
2147 | return V; | ||||||
2148 | |||||||
2149 | if (auto *VC = dyn_cast<Constant>(V)) { | ||||||
2150 | return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy), | ||||||
2151 | Name); | ||||||
2152 | } | ||||||
2153 | |||||||
2154 | return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy), | ||||||
2155 | Name); | ||||||
2156 | } | ||||||
2157 | |||||||
2158 | Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned, | ||||||
2159 | const Twine &Name = "") { | ||||||
2160 | if (V->getType() == DestTy) | ||||||
2161 | return V; | ||||||
2162 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2163 | return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name); | ||||||
2164 | return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name); | ||||||
2165 | } | ||||||
2166 | |||||||
2167 | Value *CreateBitOrPointerCast(Value *V, Type *DestTy, | ||||||
2168 | const Twine &Name = "") { | ||||||
2169 | if (V->getType() == DestTy) | ||||||
2170 | return V; | ||||||
2171 | if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy()) | ||||||
2172 | return CreatePtrToInt(V, DestTy, Name); | ||||||
2173 | if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy()) | ||||||
2174 | return CreateIntToPtr(V, DestTy, Name); | ||||||
2175 | |||||||
2176 | return CreateBitCast(V, DestTy, Name); | ||||||
2177 | } | ||||||
2178 | |||||||
2179 | Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
2180 | if (V->getType() == DestTy) | ||||||
2181 | return V; | ||||||
2182 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2183 | return Insert(Folder.CreateFPCast(VC, DestTy), Name); | ||||||
2184 | return Insert(CastInst::CreateFPCast(V, DestTy), Name); | ||||||
2185 | } | ||||||
2186 | |||||||
2187 | CallInst *CreateConstrainedFPCast( | ||||||
2188 | Intrinsic::ID ID, Value *V, Type *DestTy, | ||||||
2189 | Instruction *FMFSource = nullptr, const Twine &Name = "", | ||||||
2190 | MDNode *FPMathTag = nullptr, | ||||||
2191 | Optional<fp::RoundingMode> Rounding = None, | ||||||
2192 | Optional<fp::ExceptionBehavior> Except = None) { | ||||||
2193 | Value *ExceptV = getConstrainedFPExcept(Except); | ||||||
2194 | |||||||
2195 | FastMathFlags UseFMF = FMF; | ||||||
2196 | if (FMFSource) | ||||||
2197 | UseFMF = FMFSource->getFastMathFlags(); | ||||||
2198 | |||||||
2199 | CallInst *C; | ||||||
2200 | bool HasRoundingMD = false; | ||||||
2201 | switch (ID) { | ||||||
2202 | default: | ||||||
2203 | break; | ||||||
2204 | #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | ||||||
2205 | case Intrinsic::INTRINSIC: \ | ||||||
2206 | HasRoundingMD = ROUND_MODE; \ | ||||||
2207 | break; | ||||||
2208 | #include "llvm/IR/ConstrainedOps.def" | ||||||
2209 | } | ||||||
2210 | if (HasRoundingMD) { | ||||||
2211 | Value *RoundingV = getConstrainedFPRounding(Rounding); | ||||||
2212 | C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, | ||||||
2213 | nullptr, Name); | ||||||
2214 | } else | ||||||
2215 | C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, | ||||||
2216 | Name); | ||||||
2217 | |||||||
2218 | setConstrainedFPCallAttr(C); | ||||||
2219 | |||||||
2220 | if (isa<FPMathOperator>(C)) | ||||||
2221 | setFPAttrs(C, FPMathTag, UseFMF); | ||||||
2222 | return C; | ||||||
2223 | } | ||||||
2224 | |||||||
2225 | // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a | ||||||
2226 | // compile time error, instead of converting the string to bool for the | ||||||
2227 | // isSigned parameter. | ||||||
2228 | Value *CreateIntCast(Value *, Type *, const char *) = delete; | ||||||
2229 | |||||||
2230 | //===--------------------------------------------------------------------===// | ||||||
2231 | // Instruction creation methods: Compare Instructions | ||||||
2232 | //===--------------------------------------------------------------------===// | ||||||
2233 | |||||||
2234 | Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2235 | return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name); | ||||||
2236 | } | ||||||
2237 | |||||||
2238 | Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2239 | return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name); | ||||||
2240 | } | ||||||
2241 | |||||||
2242 | Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2243 | return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name); | ||||||
2244 | } | ||||||
2245 | |||||||
2246 | Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2247 | return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name); | ||||||
2248 | } | ||||||
2249 | |||||||
2250 | Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2251 | return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name); | ||||||
2252 | } | ||||||
2253 | |||||||
2254 | Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2255 | return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name); | ||||||
2256 | } | ||||||
2257 | |||||||
2258 | Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2259 | return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name); | ||||||
2260 | } | ||||||
2261 | |||||||
2262 | Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2263 | return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name); | ||||||
2264 | } | ||||||
2265 | |||||||
2266 | Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2267 | return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name); | ||||||
2268 | } | ||||||
2269 | |||||||
2270 | Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2271 | return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name); | ||||||
2272 | } | ||||||
2273 | |||||||
2274 | Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2275 | MDNode *FPMathTag = nullptr) { | ||||||
2276 | return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag); | ||||||
2277 | } | ||||||
2278 | |||||||
2279 | Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2280 | MDNode *FPMathTag = nullptr) { | ||||||
2281 | return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag); | ||||||
2282 | } | ||||||
2283 | |||||||
2284 | Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2285 | MDNode *FPMathTag = nullptr) { | ||||||
2286 | return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag); | ||||||
2287 | } | ||||||
2288 | |||||||
2289 | Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2290 | MDNode *FPMathTag = nullptr) { | ||||||
2291 | return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag); | ||||||
2292 | } | ||||||
2293 | |||||||
2294 | Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2295 | MDNode *FPMathTag = nullptr) { | ||||||
2296 | return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag); | ||||||
2297 | } | ||||||
2298 | |||||||
2299 | Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2300 | MDNode *FPMathTag = nullptr) { | ||||||
2301 | return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag); | ||||||
2302 | } | ||||||
2303 | |||||||
2304 | Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2305 | MDNode *FPMathTag = nullptr) { | ||||||
2306 | return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag); | ||||||
2307 | } | ||||||
2308 | |||||||
2309 | Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2310 | MDNode *FPMathTag = nullptr) { | ||||||
2311 | return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag); | ||||||
2312 | } | ||||||
2313 | |||||||
2314 | Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2315 | MDNode *FPMathTag = nullptr) { | ||||||
2316 | return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag); | ||||||
2317 | } | ||||||
2318 | |||||||
2319 | Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2320 | MDNode *FPMathTag = nullptr) { | ||||||
2321 | return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag); | ||||||
2322 | } | ||||||
2323 | |||||||
2324 | Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2325 | MDNode *FPMathTag = nullptr) { | ||||||
2326 | return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag); | ||||||
2327 | } | ||||||
2328 | |||||||
2329 | Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2330 | MDNode *FPMathTag = nullptr) { | ||||||
2331 | return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag); | ||||||
2332 | } | ||||||
2333 | |||||||
2334 | Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2335 | MDNode *FPMathTag = nullptr) { | ||||||
2336 | return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag); | ||||||
2337 | } | ||||||
2338 | |||||||
2339 | Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2340 | MDNode *FPMathTag = nullptr) { | ||||||
2341 | return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag); | ||||||
2342 | } | ||||||
2343 | |||||||
2344 | Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, | ||||||
2345 | const Twine &Name = "") { | ||||||
2346 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
2347 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
2348 | return Insert(Folder.CreateICmp(P, LC, RC), Name); | ||||||
2349 | return Insert(new ICmpInst(P, LHS, RHS), Name); | ||||||
2350 | } | ||||||
2351 | |||||||
2352 | Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, | ||||||
2353 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2354 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
2355 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
2356 | return Insert(Folder.CreateFCmp(P, LC, RC), Name); | ||||||
2357 | return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); | ||||||
2358 | } | ||||||
2359 | |||||||
2360 | //===--------------------------------------------------------------------===// | ||||||
2361 | // Instruction creation methods: Other Instructions | ||||||
2362 | //===--------------------------------------------------------------------===// | ||||||
2363 | |||||||
2364 | PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues, | ||||||
2365 | const Twine &Name = "") { | ||||||
2366 | PHINode *Phi = PHINode::Create(Ty, NumReservedValues); | ||||||
2367 | if (isa<FPMathOperator>(Phi)) | ||||||
2368 | setFPAttrs(Phi, nullptr /* MDNode* */, FMF); | ||||||
2369 | return Insert(Phi, Name); | ||||||
2370 | } | ||||||
2371 | |||||||
2372 | CallInst *CreateCall(FunctionType *FTy, Value *Callee, | ||||||
2373 | ArrayRef<Value *> Args = None, const Twine &Name = "", | ||||||
2374 | MDNode *FPMathTag = nullptr) { | ||||||
2375 | CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles); | ||||||
2376 | if (IsFPConstrained) | ||||||
2377 | setConstrainedFPCallAttr(CI); | ||||||
2378 | if (isa<FPMathOperator>(CI)) | ||||||
2379 | setFPAttrs(CI, FPMathTag, FMF); | ||||||
2380 | return Insert(CI, Name); | ||||||
2381 | } | ||||||
2382 | |||||||
2383 | CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args, | ||||||
2384 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
2385 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2386 | CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles); | ||||||
2387 | if (IsFPConstrained) | ||||||
2388 | setConstrainedFPCallAttr(CI); | ||||||
2389 | if (isa<FPMathOperator>(CI)) | ||||||
2390 | setFPAttrs(CI, FPMathTag, FMF); | ||||||
2391 | return Insert(CI, Name); | ||||||
2392 | } | ||||||
2393 | |||||||
2394 | CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None, | ||||||
2395 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2396 | return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name, | ||||||
2397 | FPMathTag); | ||||||
2398 | } | ||||||
2399 | |||||||
2400 | CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args, | ||||||
2401 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
2402 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2403 | return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, | ||||||
2404 | OpBundles, Name, FPMathTag); | ||||||
2405 | } | ||||||
2406 | |||||||
2407 | // Deprecated [opaque pointer types] | ||||||
2408 | CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args = None, | ||||||
2409 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2410 | return CreateCall( | ||||||
2411 | cast<FunctionType>(Callee->getType()->getPointerElementType()), Callee, | ||||||
2412 | Args, Name, FPMathTag); | ||||||
2413 | } | ||||||
2414 | |||||||
2415 | // Deprecated [opaque pointer types] | ||||||
2416 | CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args, | ||||||
2417 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
2418 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2419 | return CreateCall( | ||||||
2420 | cast<FunctionType>(Callee->getType()->getPointerElementType()), Callee, | ||||||
2421 | Args, OpBundles, Name, FPMathTag); | ||||||
2422 | } | ||||||
2423 | |||||||
2424 | CallInst *CreateConstrainedFPCall( | ||||||
2425 | Function *Callee, ArrayRef<Value *> Args, const Twine &Name = "", | ||||||
2426 | Optional<fp::RoundingMode> Rounding = None, | ||||||
2427 | Optional<fp::ExceptionBehavior> Except = None) { | ||||||
2428 | llvm::SmallVector<Value *, 6> UseArgs; | ||||||
2429 | |||||||
2430 | for (auto *OneArg : Args) | ||||||
2431 | UseArgs.push_back(OneArg); | ||||||
2432 | bool HasRoundingMD = false; | ||||||
2433 | switch (Callee->getIntrinsicID()) { | ||||||
2434 | default: | ||||||
2435 | break; | ||||||
2436 | #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ | ||||||
2437 | case Intrinsic::INTRINSIC: \ | ||||||
2438 | HasRoundingMD = ROUND_MODE; \ | ||||||
2439 | break; | ||||||
2440 | #include "llvm/IR/ConstrainedOps.def" | ||||||
2441 | } | ||||||
2442 | if (HasRoundingMD) | ||||||
2443 | UseArgs.push_back(getConstrainedFPRounding(Rounding)); | ||||||
2444 | UseArgs.push_back(getConstrainedFPExcept(Except)); | ||||||
2445 | |||||||
2446 | CallInst *C = CreateCall(Callee, UseArgs, Name); | ||||||
2447 | setConstrainedFPCallAttr(C); | ||||||
2448 | return C; | ||||||
2449 | } | ||||||
2450 | |||||||
2451 | Value *CreateSelect(Value *C, Value *True, Value *False, | ||||||
2452 | const Twine &Name = "", Instruction *MDFrom = nullptr) { | ||||||
2453 | if (auto *CC = dyn_cast<Constant>(C)) | ||||||
2454 | if (auto *TC = dyn_cast<Constant>(True)) | ||||||
2455 | if (auto *FC = dyn_cast<Constant>(False)) | ||||||
2456 | return Insert(Folder.CreateSelect(CC, TC, FC), Name); | ||||||
2457 | |||||||
2458 | SelectInst *Sel = SelectInst::Create(C, True, False); | ||||||
2459 | if (MDFrom) { | ||||||
2460 | MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); | ||||||
2461 | MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); | ||||||
2462 | Sel = addBranchMetadata(Sel, Prof, Unpred); | ||||||
2463 | } | ||||||
2464 | if (isa<FPMathOperator>(Sel)) | ||||||
2465 | setFPAttrs(Sel, nullptr /* MDNode* */, FMF); | ||||||
2466 | return Insert(Sel, Name); | ||||||
2467 | } | ||||||
2468 | |||||||
2469 | VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") { | ||||||
2470 | return Insert(new VAArgInst(List, Ty), Name); | ||||||
2471 | } | ||||||
2472 | |||||||
2473 | Value *CreateExtractElement(Value *Vec, Value *Idx, | ||||||
2474 | const Twine &Name = "") { | ||||||
2475 | if (auto *VC = dyn_cast<Constant>(Vec)) | ||||||
2476 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
2477 | return Insert(Folder.CreateExtractElement(VC, IC), Name); | ||||||
2478 | return Insert(ExtractElementInst::Create(Vec, Idx), Name); | ||||||
2479 | } | ||||||
2480 | |||||||
2481 | Value *CreateExtractElement(Value *Vec, uint64_t Idx, | ||||||
2482 | const Twine &Name = "") { | ||||||
2483 | return CreateExtractElement(Vec, getInt64(Idx), Name); | ||||||
2484 | } | ||||||
2485 | |||||||
2486 | Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx, | ||||||
2487 | const Twine &Name = "") { | ||||||
2488 | if (auto *VC = dyn_cast<Constant>(Vec)) | ||||||
2489 | if (auto *NC = dyn_cast<Constant>(NewElt)) | ||||||
2490 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
2491 | return Insert(Folder.CreateInsertElement(VC, NC, IC), Name); | ||||||
2492 | return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name); | ||||||
2493 | } | ||||||
2494 | |||||||
2495 | Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx, | ||||||
2496 | const Twine &Name = "") { | ||||||
2497 | return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name); | ||||||
2498 | } | ||||||
2499 | |||||||
2500 | Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask, | ||||||
2501 | const Twine &Name = "") { | ||||||
2502 | if (auto *V1C = dyn_cast<Constant>(V1)) | ||||||
2503 | if (auto *V2C = dyn_cast<Constant>(V2)) | ||||||
2504 | if (auto *MC = dyn_cast<Constant>(Mask)) | ||||||
2505 | return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name); | ||||||
2506 | return Insert(new ShuffleVectorInst(V1, V2, Mask), Name); | ||||||
2507 | } | ||||||
2508 | |||||||
2509 | Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask, | ||||||
2510 | const Twine &Name = "") { | ||||||
2511 | Value *Mask = ConstantDataVector::get(Context, IntMask); | ||||||
2512 | return CreateShuffleVector(V1, V2, Mask, Name); | ||||||
2513 | } | ||||||
2514 | |||||||
2515 | Value *CreateExtractValue(Value *Agg, | ||||||
2516 | ArrayRef<unsigned> Idxs, | ||||||
2517 | const Twine &Name = "") { | ||||||
2518 | if (auto *AggC = dyn_cast<Constant>(Agg)) | ||||||
2519 | return Insert(Folder.CreateExtractValue(AggC, Idxs), Name); | ||||||
2520 | return Insert(ExtractValueInst::Create(Agg, Idxs), Name); | ||||||
2521 | } | ||||||
2522 | |||||||
2523 | Value *CreateInsertValue(Value *Agg, Value *Val, | ||||||
2524 | ArrayRef<unsigned> Idxs, | ||||||
2525 | const Twine &Name = "") { | ||||||
2526 | if (auto *AggC = dyn_cast<Constant>(Agg)) | ||||||
2527 | if (auto *ValC = dyn_cast<Constant>(Val)) | ||||||
2528 | return Insert(Folder.CreateInsertValue(AggC, ValC, Idxs), Name); | ||||||
2529 | return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name); | ||||||
2530 | } | ||||||
2531 | |||||||
2532 | LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses, | ||||||
2533 | const Twine &Name = "") { | ||||||
2534 | return Insert(LandingPadInst::Create(Ty, NumClauses), Name); | ||||||
2535 | } | ||||||
2536 | |||||||
2537 | Value *CreateFreeze(Value *V, const Twine &Name = "") { | ||||||
2538 | return Insert(new FreezeInst(V), Name); | ||||||
2539 | } | ||||||
2540 | |||||||
2541 | //===--------------------------------------------------------------------===// | ||||||
2542 | // Utility creation methods | ||||||
2543 | //===--------------------------------------------------------------------===// | ||||||
2544 | |||||||
2545 | /// Return an i1 value testing if \p Arg is null. | ||||||
2546 | Value *CreateIsNull(Value *Arg, const Twine &Name = "") { | ||||||
2547 | return CreateICmpEQ(Arg, Constant::getNullValue(Arg->getType()), | ||||||
2548 | Name); | ||||||
2549 | } | ||||||
2550 | |||||||
2551 | /// Return an i1 value testing if \p Arg is not null. | ||||||
2552 | Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") { | ||||||
2553 | return CreateICmpNE(Arg, Constant::getNullValue(Arg->getType()), | ||||||
2554 | Name); | ||||||
2555 | } | ||||||
2556 | |||||||
2557 | /// Return the i64 difference between two pointer values, dividing out | ||||||
2558 | /// the size of the pointed-to objects. | ||||||
2559 | /// | ||||||
2560 | /// This is intended to implement C-style pointer subtraction. As such, the | ||||||
2561 | /// pointers must be appropriately aligned for their element types and | ||||||
2562 | /// pointing into the same object. | ||||||
2563 | Value *CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2564 | assert(LHS->getType() == RHS->getType() &&((LHS->getType() == RHS->getType() && "Pointer subtraction operand types must match!" ) ? static_cast<void> (0) : __assert_fail ("LHS->getType() == RHS->getType() && \"Pointer subtraction operand types must match!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2565, __PRETTY_FUNCTION__)) | ||||||
2565 | "Pointer subtraction operand types must match!")((LHS->getType() == RHS->getType() && "Pointer subtraction operand types must match!" ) ? static_cast<void> (0) : __assert_fail ("LHS->getType() == RHS->getType() && \"Pointer subtraction operand types must match!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2565, __PRETTY_FUNCTION__)); | ||||||
2566 | auto *ArgType = cast<PointerType>(LHS->getType()); | ||||||
2567 | Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); | ||||||
2568 | Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); | ||||||
2569 | Value *Difference = CreateSub(LHS_int, RHS_int); | ||||||
2570 | return CreateExactSDiv(Difference, | ||||||
2571 | ConstantExpr::getSizeOf(ArgType->getElementType()), | ||||||
2572 | Name); | ||||||
2573 | } | ||||||
2574 | |||||||
2575 | /// Create a launder.invariant.group intrinsic call. If Ptr type is | ||||||
2576 | /// different from pointer to i8, it's casted to pointer to i8 in the same | ||||||
2577 | /// address space before call and casted back to Ptr type after call. | ||||||
2578 | Value *CreateLaunderInvariantGroup(Value *Ptr) { | ||||||
2579 | assert(isa<PointerType>(Ptr->getType()) &&((isa<PointerType>(Ptr->getType()) && "launder.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"launder.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2580, __PRETTY_FUNCTION__)) | ||||||
2580 | "launder.invariant.group only applies to pointers.")((isa<PointerType>(Ptr->getType()) && "launder.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"launder.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2580, __PRETTY_FUNCTION__)); | ||||||
2581 | // FIXME: we could potentially avoid casts to/from i8*. | ||||||
2582 | auto *PtrType = Ptr->getType(); | ||||||
2583 | auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); | ||||||
2584 | if (PtrType != Int8PtrTy) | ||||||
2585 | Ptr = CreateBitCast(Ptr, Int8PtrTy); | ||||||
2586 | Module *M = BB->getParent()->getParent(); | ||||||
2587 | Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( | ||||||
2588 | M, Intrinsic::launder_invariant_group, {Int8PtrTy}); | ||||||
2589 | |||||||
2590 | assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2593, __PRETTY_FUNCTION__)) | ||||||
2591 | FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2593, __PRETTY_FUNCTION__)) | ||||||
2592 | Int8PtrTy &&((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2593, __PRETTY_FUNCTION__)) | ||||||
2593 | "LaunderInvariantGroup should take and return the same type")((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2593, __PRETTY_FUNCTION__)); | ||||||
2594 | |||||||
2595 | CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr}); | ||||||
2596 | |||||||
2597 | if (PtrType != Int8PtrTy) | ||||||
2598 | return CreateBitCast(Fn, PtrType); | ||||||
2599 | return Fn; | ||||||
2600 | } | ||||||
2601 | |||||||
2602 | /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is | ||||||
2603 | /// different from pointer to i8, it's casted to pointer to i8 in the same | ||||||
2604 | /// address space before call and casted back to Ptr type after call. | ||||||
2605 | Value *CreateStripInvariantGroup(Value *Ptr) { | ||||||
2606 | assert(isa<PointerType>(Ptr->getType()) &&((isa<PointerType>(Ptr->getType()) && "strip.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"strip.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2607, __PRETTY_FUNCTION__)) | ||||||
2607 | "strip.invariant.group only applies to pointers.")((isa<PointerType>(Ptr->getType()) && "strip.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"strip.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2607, __PRETTY_FUNCTION__)); | ||||||
2608 | |||||||
2609 | // FIXME: we could potentially avoid casts to/from i8*. | ||||||
2610 | auto *PtrType = Ptr->getType(); | ||||||
2611 | auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); | ||||||
2612 | if (PtrType != Int8PtrTy) | ||||||
2613 | Ptr = CreateBitCast(Ptr, Int8PtrTy); | ||||||
2614 | Module *M = BB->getParent()->getParent(); | ||||||
2615 | Function *FnStripInvariantGroup = Intrinsic::getDeclaration( | ||||||
2616 | M, Intrinsic::strip_invariant_group, {Int8PtrTy}); | ||||||
2617 | |||||||
2618 | assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2621, __PRETTY_FUNCTION__)) | ||||||
2619 | FnStripInvariantGroup->getFunctionType()->getParamType(0) ==((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2621, __PRETTY_FUNCTION__)) | ||||||
2620 | Int8PtrTy &&((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2621, __PRETTY_FUNCTION__)) | ||||||
2621 | "StripInvariantGroup should take and return the same type")((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2621, __PRETTY_FUNCTION__)); | ||||||
2622 | |||||||
2623 | CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr}); | ||||||
2624 | |||||||
2625 | if (PtrType != Int8PtrTy) | ||||||
2626 | return CreateBitCast(Fn, PtrType); | ||||||
2627 | return Fn; | ||||||
2628 | } | ||||||
2629 | |||||||
2630 | /// Return a vector value that contains \arg V broadcasted to \p | ||||||
2631 | /// NumElts elements. | ||||||
2632 | Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "") { | ||||||
2633 | assert(NumElts > 0 && "Cannot splat to an empty vector!")((NumElts > 0 && "Cannot splat to an empty vector!" ) ? static_cast<void> (0) : __assert_fail ("NumElts > 0 && \"Cannot splat to an empty vector!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2633, __PRETTY_FUNCTION__)); | ||||||
2634 | |||||||
2635 | // First insert it into an undef vector so we can shuffle it. | ||||||
2636 | Type *I32Ty = getInt32Ty(); | ||||||
2637 | Value *Undef = UndefValue::get(VectorType::get(V->getType(), NumElts)); | ||||||
2638 | V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0), | ||||||
2639 | Name + ".splatinsert"); | ||||||
2640 | |||||||
2641 | // Shuffle the value across the desired number of elements. | ||||||
2642 | Value *Zeros = ConstantAggregateZero::get(VectorType::get(I32Ty, NumElts)); | ||||||
2643 | return CreateShuffleVector(V, Undef, Zeros, Name + ".splat"); | ||||||
2644 | } | ||||||
2645 | |||||||
2646 | /// Return a value that has been extracted from a larger integer type. | ||||||
2647 | Value *CreateExtractInteger(const DataLayout &DL, Value *From, | ||||||
2648 | IntegerType *ExtractedTy, uint64_t Offset, | ||||||
2649 | const Twine &Name) { | ||||||
2650 | auto *IntTy = cast<IntegerType>(From->getType()); | ||||||
2651 | assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=((DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize (IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2653, __PRETTY_FUNCTION__)) | ||||||
2652 | DL.getTypeStoreSize(IntTy) &&((DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize (IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2653, __PRETTY_FUNCTION__)) | ||||||
2653 | "Element extends past full value")((DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize (IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2653, __PRETTY_FUNCTION__)); | ||||||
2654 | uint64_t ShAmt = 8 * Offset; | ||||||
2655 | Value *V = From; | ||||||
2656 | if (DL.isBigEndian()) | ||||||
2657 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - | ||||||
2658 | DL.getTypeStoreSize(ExtractedTy) - Offset); | ||||||
2659 | if (ShAmt) { | ||||||
2660 | V = CreateLShr(V, ShAmt, Name + ".shift"); | ||||||
2661 | } | ||||||
2662 | assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&((ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast <void> (0) : __assert_fail ("ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2663, __PRETTY_FUNCTION__)) | ||||||
2663 | "Cannot extract to a larger integer!")((ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast <void> (0) : __assert_fail ("ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2663, __PRETTY_FUNCTION__)); | ||||||
2664 | if (ExtractedTy != IntTy) { | ||||||
2665 | V = CreateTrunc(V, ExtractedTy, Name + ".trunc"); | ||||||
2666 | } | ||||||
2667 | return V; | ||||||
2668 | } | ||||||
2669 | |||||||
2670 | Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base, | ||||||
2671 | unsigned Dimension, unsigned LastIndex, | ||||||
2672 | MDNode *DbgInfo) { | ||||||
2673 | assert(isa<PointerType>(Base->getType()) &&((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.array.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.array.access.index.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2674, __PRETTY_FUNCTION__)) | ||||||
2674 | "Invalid Base ptr type for preserve.array.access.index.")((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.array.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.array.access.index.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2674, __PRETTY_FUNCTION__)); | ||||||
2675 | auto *BaseType = Base->getType(); | ||||||
2676 | |||||||
2677 | Value *LastIndexV = getInt32(LastIndex); | ||||||
2678 | Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); | ||||||
2679 | SmallVector<Value *, 4> IdxList; | ||||||
2680 | for (unsigned I = 0; I < Dimension; ++I) | ||||||
2681 | IdxList.push_back(Zero); | ||||||
2682 | IdxList.push_back(LastIndexV); | ||||||
2683 | |||||||
2684 | Type *ResultType = | ||||||
2685 | GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); | ||||||
2686 | |||||||
2687 | Module *M = BB->getParent()->getParent(); | ||||||
2688 | Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( | ||||||
2689 | M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); | ||||||
2690 | |||||||
2691 | Value *DimV = getInt32(Dimension); | ||||||
2692 | CallInst *Fn = | ||||||
2693 | CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); | ||||||
2694 | if (DbgInfo) | ||||||
2695 | Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); | ||||||
2696 | |||||||
2697 | return Fn; | ||||||
2698 | } | ||||||
2699 | |||||||
2700 | Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex, | ||||||
2701 | MDNode *DbgInfo) { | ||||||
2702 | assert(isa<PointerType>(Base->getType()) &&((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.union.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.union.access.index.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2703, __PRETTY_FUNCTION__)) | ||||||
2703 | "Invalid Base ptr type for preserve.union.access.index.")((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.union.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.union.access.index.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2703, __PRETTY_FUNCTION__)); | ||||||
2704 | auto *BaseType = Base->getType(); | ||||||
2705 | |||||||
2706 | Module *M = BB->getParent()->getParent(); | ||||||
2707 | Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( | ||||||
2708 | M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); | ||||||
2709 | |||||||
2710 | Value *DIIndex = getInt32(FieldIndex); | ||||||
2711 | CallInst *Fn = | ||||||
2712 | CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); | ||||||
2713 | if (DbgInfo) | ||||||
2714 | Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); | ||||||
2715 | |||||||
2716 | return Fn; | ||||||
2717 | } | ||||||
2718 | |||||||
2719 | Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base, | ||||||
2720 | unsigned Index, unsigned FieldIndex, | ||||||
2721 | MDNode *DbgInfo) { | ||||||
2722 | assert(isa<PointerType>(Base->getType()) &&((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.struct.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.struct.access.index.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2723, __PRETTY_FUNCTION__)) | ||||||
2723 | "Invalid Base ptr type for preserve.struct.access.index.")((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.struct.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.struct.access.index.\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2723, __PRETTY_FUNCTION__)); | ||||||
2724 | auto *BaseType = Base->getType(); | ||||||
2725 | |||||||
2726 | Value *GEPIndex = getInt32(Index); | ||||||
2727 | Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); | ||||||
2728 | Type *ResultType = | ||||||
2729 | GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); | ||||||
2730 | |||||||
2731 | Module *M = BB->getParent()->getParent(); | ||||||
2732 | Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( | ||||||
2733 | M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); | ||||||
2734 | |||||||
2735 | Value *DIIndex = getInt32(FieldIndex); | ||||||
2736 | CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, | ||||||
2737 | {Base, GEPIndex, DIIndex}); | ||||||
2738 | if (DbgInfo) | ||||||
2739 | Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); | ||||||
2740 | |||||||
2741 | return Fn; | ||||||
2742 | } | ||||||
2743 | |||||||
2744 | private: | ||||||
2745 | /// Helper function that creates an assume intrinsic call that | ||||||
2746 | /// represents an alignment assumption on the provided Ptr, Mask, Type | ||||||
2747 | /// and Offset. It may be sometimes useful to do some other logic | ||||||
2748 | /// based on this alignment check, thus it can be stored into 'TheCheck'. | ||||||
2749 | CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL, | ||||||
2750 | Value *PtrValue, Value *Mask, | ||||||
2751 | Type *IntPtrTy, Value *OffsetValue, | ||||||
2752 | Value **TheCheck) { | ||||||
2753 | Value *PtrIntValue = CreatePtrToInt(PtrValue, IntPtrTy, "ptrint"); | ||||||
2754 | |||||||
2755 | if (OffsetValue) { | ||||||
2756 | bool IsOffsetZero = false; | ||||||
2757 | if (const auto *CI = dyn_cast<ConstantInt>(OffsetValue)) | ||||||
2758 | IsOffsetZero = CI->isZero(); | ||||||
2759 | |||||||
2760 | if (!IsOffsetZero) { | ||||||
2761 | if (OffsetValue->getType() != IntPtrTy) | ||||||
2762 | OffsetValue = CreateIntCast(OffsetValue, IntPtrTy, /*isSigned*/ true, | ||||||
2763 | "offsetcast"); | ||||||
2764 | PtrIntValue = CreateSub(PtrIntValue, OffsetValue, "offsetptr"); | ||||||
2765 | } | ||||||
2766 | } | ||||||
2767 | |||||||
2768 | Value *Zero = ConstantInt::get(IntPtrTy, 0); | ||||||
2769 | Value *MaskedPtr = CreateAnd(PtrIntValue, Mask, "maskedptr"); | ||||||
2770 | Value *InvCond = CreateICmpEQ(MaskedPtr, Zero, "maskcond"); | ||||||
2771 | if (TheCheck) | ||||||
2772 | *TheCheck = InvCond; | ||||||
2773 | |||||||
2774 | return CreateAssumption(InvCond); | ||||||
2775 | } | ||||||
2776 | |||||||
2777 | public: | ||||||
2778 | /// Create an assume intrinsic call that represents an alignment | ||||||
2779 | /// assumption on the provided pointer. | ||||||
2780 | /// | ||||||
2781 | /// An optional offset can be provided, and if it is provided, the offset | ||||||
2782 | /// must be subtracted from the provided pointer to get the pointer with the | ||||||
2783 | /// specified alignment. | ||||||
2784 | /// | ||||||
2785 | /// It may be sometimes useful to do some other logic | ||||||
2786 | /// based on this alignment check, thus it can be stored into 'TheCheck'. | ||||||
2787 | CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, | ||||||
2788 | unsigned Alignment, | ||||||
2789 | Value *OffsetValue = nullptr, | ||||||
2790 | Value **TheCheck = nullptr) { | ||||||
2791 | assert(isa<PointerType>(PtrValue->getType()) &&((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2792, __PRETTY_FUNCTION__)) | ||||||
2792 | "trying to create an alignment assumption on a non-pointer?")((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2792, __PRETTY_FUNCTION__)); | ||||||
2793 | assert(Alignment != 0 && "Invalid Alignment")((Alignment != 0 && "Invalid Alignment") ? static_cast <void> (0) : __assert_fail ("Alignment != 0 && \"Invalid Alignment\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2793, __PRETTY_FUNCTION__)); | ||||||
2794 | auto *PtrTy = cast<PointerType>(PtrValue->getType()); | ||||||
2795 | Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); | ||||||
2796 | |||||||
2797 | Value *Mask = ConstantInt::get(IntPtrTy, Alignment - 1); | ||||||
2798 | return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, | ||||||
2799 | OffsetValue, TheCheck); | ||||||
2800 | } | ||||||
2801 | |||||||
2802 | /// Create an assume intrinsic call that represents an alignment | ||||||
2803 | /// assumption on the provided pointer. | ||||||
2804 | /// | ||||||
2805 | /// An optional offset can be provided, and if it is provided, the offset | ||||||
2806 | /// must be subtracted from the provided pointer to get the pointer with the | ||||||
2807 | /// specified alignment. | ||||||
2808 | /// | ||||||
2809 | /// It may be sometimes useful to do some other logic | ||||||
2810 | /// based on this alignment check, thus it can be stored into 'TheCheck'. | ||||||
2811 | /// | ||||||
2812 | /// This overload handles the condition where the Alignment is dependent | ||||||
2813 | /// on an existing value rather than a static value. | ||||||
2814 | CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, | ||||||
2815 | Value *Alignment, | ||||||
2816 | Value *OffsetValue = nullptr, | ||||||
2817 | Value **TheCheck = nullptr) { | ||||||
2818 | assert(isa<PointerType>(PtrValue->getType()) &&((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2819, __PRETTY_FUNCTION__)) | ||||||
2819 | "trying to create an alignment assumption on a non-pointer?")((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h" , 2819, __PRETTY_FUNCTION__)); | ||||||
2820 | auto *PtrTy = cast<PointerType>(PtrValue->getType()); | ||||||
2821 | Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); | ||||||
2822 | |||||||
2823 | if (Alignment->getType() != IntPtrTy) | ||||||
2824 | Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ false, | ||||||
2825 | "alignmentcast"); | ||||||
2826 | |||||||
2827 | Value *Mask = CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "mask"); | ||||||
2828 | |||||||
2829 | return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, | ||||||
2830 | OffsetValue, TheCheck); | ||||||
2831 | } | ||||||
2832 | }; | ||||||
2833 | |||||||
2834 | // Create wrappers for C Binding types (see CBindingWrapping.h). | ||||||
2835 | DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast <IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef >(const_cast<IRBuilder<>*>(P)); } | ||||||
2836 | |||||||
2837 | } // end namespace llvm | ||||||
2838 | |||||||
2839 | #endif // LLVM_IR_IRBUILDER_H |