Bug Summary

File:llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
Warning:line 1140, column 32
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name MemorySanitizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/lib/Transforms/Instrumentation -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-11-115256-23437-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp

1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15/// byte of the application memory, poison the shadow of the malloc-ed
16/// or alloca-ed memory, load the shadow bits on every memory read,
17/// propagate the shadow bits through some of the arithmetic
18/// instruction (including MOV), store the shadow bits on every memory
19/// write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwritting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// - TLS variables are stored in a single per-task struct. A call to a
126/// function __msan_get_context_state() returning a pointer to that struct
127/// is inserted into every instrumented function before the entry block;
128/// - __msan_warning() takes a 32-bit origin parameter;
129/// - local variables are poisoned with __msan_poison_alloca() upon function
130/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131/// function;
132/// - the pass doesn't declare any global variables or add global constructors
133/// to the translation unit.
134///
135/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136/// calls, making sure we're on the safe side wrt. possible false positives.
137///
138/// KernelMemorySanitizer only supports X86_64 at the moment.
139///
140//===----------------------------------------------------------------------===//
141
142#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
143#include "llvm/ADT/APInt.h"
144#include "llvm/ADT/ArrayRef.h"
145#include "llvm/ADT/DepthFirstIterator.h"
146#include "llvm/ADT/SmallSet.h"
147#include "llvm/ADT/SmallString.h"
148#include "llvm/ADT/SmallVector.h"
149#include "llvm/ADT/StringExtras.h"
150#include "llvm/ADT/StringRef.h"
151#include "llvm/ADT/Triple.h"
152#include "llvm/Analysis/TargetLibraryInfo.h"
153#include "llvm/IR/Argument.h"
154#include "llvm/IR/Attributes.h"
155#include "llvm/IR/BasicBlock.h"
156#include "llvm/IR/CallSite.h"
157#include "llvm/IR/CallingConv.h"
158#include "llvm/IR/Constant.h"
159#include "llvm/IR/Constants.h"
160#include "llvm/IR/DataLayout.h"
161#include "llvm/IR/DerivedTypes.h"
162#include "llvm/IR/Function.h"
163#include "llvm/IR/GlobalValue.h"
164#include "llvm/IR/GlobalVariable.h"
165#include "llvm/IR/IRBuilder.h"
166#include "llvm/IR/InlineAsm.h"
167#include "llvm/IR/InstVisitor.h"
168#include "llvm/IR/InstrTypes.h"
169#include "llvm/IR/Instruction.h"
170#include "llvm/IR/Instructions.h"
171#include "llvm/IR/IntrinsicInst.h"
172#include "llvm/IR/Intrinsics.h"
173#include "llvm/IR/IntrinsicsX86.h"
174#include "llvm/IR/LLVMContext.h"
175#include "llvm/IR/MDBuilder.h"
176#include "llvm/IR/Module.h"
177#include "llvm/IR/Type.h"
178#include "llvm/IR/Value.h"
179#include "llvm/IR/ValueMap.h"
180#include "llvm/InitializePasses.h"
181#include "llvm/Pass.h"
182#include "llvm/Support/AtomicOrdering.h"
183#include "llvm/Support/Casting.h"
184#include "llvm/Support/CommandLine.h"
185#include "llvm/Support/Compiler.h"
186#include "llvm/Support/Debug.h"
187#include "llvm/Support/ErrorHandling.h"
188#include "llvm/Support/MathExtras.h"
189#include "llvm/Support/raw_ostream.h"
190#include "llvm/Transforms/Instrumentation.h"
191#include "llvm/Transforms/Utils/BasicBlockUtils.h"
192#include "llvm/Transforms/Utils/Local.h"
193#include "llvm/Transforms/Utils/ModuleUtils.h"
194#include <algorithm>
195#include <cassert>
196#include <cstddef>
197#include <cstdint>
198#include <memory>
199#include <string>
200#include <tuple>
201
202using namespace llvm;
203
204#define DEBUG_TYPE"msan" "msan"
205
206static const unsigned kOriginSize = 4;
207static const Align kMinOriginAlignment = Align(4);
208static const Align kShadowTLSAlignment = Align(8);
209
210// These constants must be kept in sync with the ones in msan.h.
211static const unsigned kParamTLSSize = 800;
212static const unsigned kRetvalTLSSize = 800;
213
214// Accesses sizes are powers of two: 1, 2, 4, 8.
215static const size_t kNumberOfAccessSizes = 4;
216
217/// Track origins of uninitialized values.
218///
219/// Adds a section to MemorySanitizer report that points to the allocation
220/// (stack or heap) the uninitialized bits came from originally.
221static cl::opt<int> ClTrackOrigins("msan-track-origins",
222 cl::desc("Track origins (allocation sites) of poisoned memory"),
223 cl::Hidden, cl::init(0));
224
225static cl::opt<bool> ClKeepGoing("msan-keep-going",
226 cl::desc("keep going after reporting a UMR"),
227 cl::Hidden, cl::init(false));
228
229static cl::opt<bool> ClPoisonStack("msan-poison-stack",
230 cl::desc("poison uninitialized stack variables"),
231 cl::Hidden, cl::init(true));
232
233static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
234 cl::desc("poison uninitialized stack variables with a call"),
235 cl::Hidden, cl::init(false));
236
237static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
238 cl::desc("poison uninitialized stack variables with the given pattern"),
239 cl::Hidden, cl::init(0xff));
240
241static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
242 cl::desc("poison undef temps"),
243 cl::Hidden, cl::init(true));
244
245static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
246 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
247 cl::Hidden, cl::init(true));
248
249static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
250 cl::desc("exact handling of relational integer ICmp"),
251 cl::Hidden, cl::init(false));
252
253static cl::opt<bool> ClHandleLifetimeIntrinsics(
254 "msan-handle-lifetime-intrinsics",
255 cl::desc(
256 "when possible, poison scoped variables at the beginning of the scope "
257 "(slower, but more precise)"),
258 cl::Hidden, cl::init(true));
259
260// When compiling the Linux kernel, we sometimes see false positives related to
261// MSan being unable to understand that inline assembly calls may initialize
262// local variables.
263// This flag makes the compiler conservatively unpoison every memory location
264// passed into an assembly call. Note that this may cause false positives.
265// Because it's impossible to figure out the array sizes, we can only unpoison
266// the first sizeof(type) bytes for each type* pointer.
267// The instrumentation is only enabled in KMSAN builds, and only if
268// -msan-handle-asm-conservative is on. This is done because we may want to
269// quickly disable assembly instrumentation when it breaks.
270static cl::opt<bool> ClHandleAsmConservative(
271 "msan-handle-asm-conservative",
272 cl::desc("conservative handling of inline assembly"), cl::Hidden,
273 cl::init(true));
274
275// This flag controls whether we check the shadow of the address
276// operand of load or store. Such bugs are very rare, since load from
277// a garbage address typically results in SEGV, but still happen
278// (e.g. only lower bits of address are garbage, or the access happens
279// early at program startup where malloc-ed memory is more likely to
280// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
281static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
282 cl::desc("report accesses through a pointer which has poisoned shadow"),
283 cl::Hidden, cl::init(true));
284
285static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
286 cl::desc("print out instructions with default strict semantics"),
287 cl::Hidden, cl::init(false));
288
289static cl::opt<int> ClInstrumentationWithCallThreshold(
290 "msan-instrumentation-with-call-threshold",
291 cl::desc(
292 "If the function being instrumented requires more than "
293 "this number of checks and origin stores, use callbacks instead of "
294 "inline checks (-1 means never use callbacks)."),
295 cl::Hidden, cl::init(3500));
296
297static cl::opt<bool>
298 ClEnableKmsan("msan-kernel",
299 cl::desc("Enable KernelMemorySanitizer instrumentation"),
300 cl::Hidden, cl::init(false));
301
302// This is an experiment to enable handling of cases where shadow is a non-zero
303// compile-time constant. For some unexplainable reason they were silently
304// ignored in the instrumentation.
305static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
306 cl::desc("Insert checks for constant shadow values"),
307 cl::Hidden, cl::init(false));
308
309// This is off by default because of a bug in gold:
310// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
311static cl::opt<bool> ClWithComdat("msan-with-comdat",
312 cl::desc("Place MSan constructors in comdat sections"),
313 cl::Hidden, cl::init(false));
314
315// These options allow to specify custom memory map parameters
316// See MemoryMapParams for details.
317static cl::opt<uint64_t> ClAndMask("msan-and-mask",
318 cl::desc("Define custom MSan AndMask"),
319 cl::Hidden, cl::init(0));
320
321static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
322 cl::desc("Define custom MSan XorMask"),
323 cl::Hidden, cl::init(0));
324
325static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
326 cl::desc("Define custom MSan ShadowBase"),
327 cl::Hidden, cl::init(0));
328
329static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
330 cl::desc("Define custom MSan OriginBase"),
331 cl::Hidden, cl::init(0));
332
333static const char *const kMsanModuleCtorName = "msan.module_ctor";
334static const char *const kMsanInitName = "__msan_init";
335
336namespace {
337
338// Memory map parameters used in application-to-shadow address calculation.
339// Offset = (Addr & ~AndMask) ^ XorMask
340// Shadow = ShadowBase + Offset
341// Origin = OriginBase + Offset
342struct MemoryMapParams {
343 uint64_t AndMask;
344 uint64_t XorMask;
345 uint64_t ShadowBase;
346 uint64_t OriginBase;
347};
348
349struct PlatformMemoryMapParams {
350 const MemoryMapParams *bits32;
351 const MemoryMapParams *bits64;
352};
353
354} // end anonymous namespace
355
356// i386 Linux
357static const MemoryMapParams Linux_I386_MemoryMapParams = {
358 0x000080000000, // AndMask
359 0, // XorMask (not used)
360 0, // ShadowBase (not used)
361 0x000040000000, // OriginBase
362};
363
364// x86_64 Linux
365static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
366#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
367 0x400000000000, // AndMask
368 0, // XorMask (not used)
369 0, // ShadowBase (not used)
370 0x200000000000, // OriginBase
371#else
372 0, // AndMask (not used)
373 0x500000000000, // XorMask
374 0, // ShadowBase (not used)
375 0x100000000000, // OriginBase
376#endif
377};
378
379// mips64 Linux
380static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
381 0, // AndMask (not used)
382 0x008000000000, // XorMask
383 0, // ShadowBase (not used)
384 0x002000000000, // OriginBase
385};
386
387// ppc64 Linux
388static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
389 0xE00000000000, // AndMask
390 0x100000000000, // XorMask
391 0x080000000000, // ShadowBase
392 0x1C0000000000, // OriginBase
393};
394
395// aarch64 Linux
396static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
397 0, // AndMask (not used)
398 0x06000000000, // XorMask
399 0, // ShadowBase (not used)
400 0x01000000000, // OriginBase
401};
402
403// i386 FreeBSD
404static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
405 0x000180000000, // AndMask
406 0x000040000000, // XorMask
407 0x000020000000, // ShadowBase
408 0x000700000000, // OriginBase
409};
410
411// x86_64 FreeBSD
412static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
413 0xc00000000000, // AndMask
414 0x200000000000, // XorMask
415 0x100000000000, // ShadowBase
416 0x380000000000, // OriginBase
417};
418
419// x86_64 NetBSD
420static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
421 0, // AndMask
422 0x500000000000, // XorMask
423 0, // ShadowBase
424 0x100000000000, // OriginBase
425};
426
427static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
428 &Linux_I386_MemoryMapParams,
429 &Linux_X86_64_MemoryMapParams,
430};
431
432static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
433 nullptr,
434 &Linux_MIPS64_MemoryMapParams,
435};
436
437static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
438 nullptr,
439 &Linux_PowerPC64_MemoryMapParams,
440};
441
442static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
443 nullptr,
444 &Linux_AArch64_MemoryMapParams,
445};
446
447static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
448 &FreeBSD_I386_MemoryMapParams,
449 &FreeBSD_X86_64_MemoryMapParams,
450};
451
452static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
453 nullptr,
454 &NetBSD_X86_64_MemoryMapParams,
455};
456
457namespace {
458
459/// Instrument functions of a module to detect uninitialized reads.
460///
461/// Instantiating MemorySanitizer inserts the msan runtime library API function
462/// declarations into the module if they don't exist already. Instantiating
463/// ensures the __msan_init function is in the list of global constructors for
464/// the module.
465class MemorySanitizer {
466public:
467 MemorySanitizer(Module &M, MemorySanitizerOptions Options)
468 : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
469 Recover(Options.Recover) {
470 initializeModule(M);
471 }
472
473 // MSan cannot be moved or copied because of MapParams.
474 MemorySanitizer(MemorySanitizer &&) = delete;
475 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
476 MemorySanitizer(const MemorySanitizer &) = delete;
477 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
478
479 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
480
481private:
482 friend struct MemorySanitizerVisitor;
483 friend struct VarArgAMD64Helper;
484 friend struct VarArgMIPS64Helper;
485 friend struct VarArgAArch64Helper;
486 friend struct VarArgPowerPC64Helper;
487
488 void initializeModule(Module &M);
489 void initializeCallbacks(Module &M);
490 void createKernelApi(Module &M);
491 void createUserspaceApi(Module &M);
492
493 /// True if we're compiling the Linux kernel.
494 bool CompileKernel;
495 /// Track origins (allocation points) of uninitialized values.
496 int TrackOrigins;
497 bool Recover;
498
499 LLVMContext *C;
500 Type *IntptrTy;
501 Type *OriginTy;
502
503 // XxxTLS variables represent the per-thread state in MSan and per-task state
504 // in KMSAN.
505 // For the userspace these point to thread-local globals. In the kernel land
506 // they point to the members of a per-task struct obtained via a call to
507 // __msan_get_context_state().
508
509 /// Thread-local shadow storage for function parameters.
510 Value *ParamTLS;
511
512 /// Thread-local origin storage for function parameters.
513 Value *ParamOriginTLS;
514
515 /// Thread-local shadow storage for function return value.
516 Value *RetvalTLS;
517
518 /// Thread-local origin storage for function return value.
519 Value *RetvalOriginTLS;
520
521 /// Thread-local shadow storage for in-register va_arg function
522 /// parameters (x86_64-specific).
523 Value *VAArgTLS;
524
525 /// Thread-local shadow storage for in-register va_arg function
526 /// parameters (x86_64-specific).
527 Value *VAArgOriginTLS;
528
529 /// Thread-local shadow storage for va_arg overflow area
530 /// (x86_64-specific).
531 Value *VAArgOverflowSizeTLS;
532
533 /// Thread-local space used to pass origin value to the UMR reporting
534 /// function.
535 Value *OriginTLS;
536
537 /// Are the instrumentation callbacks set up?
538 bool CallbacksInitialized = false;
539
540 /// The run-time callback to print a warning.
541 FunctionCallee WarningFn;
542
543 // These arrays are indexed by log2(AccessSize).
544 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
545 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
546
547 /// Run-time helper that generates a new origin value for a stack
548 /// allocation.
549 FunctionCallee MsanSetAllocaOrigin4Fn;
550
551 /// Run-time helper that poisons stack on function entry.
552 FunctionCallee MsanPoisonStackFn;
553
554 /// Run-time helper that records a store (or any event) of an
555 /// uninitialized value and returns an updated origin id encoding this info.
556 FunctionCallee MsanChainOriginFn;
557
558 /// MSan runtime replacements for memmove, memcpy and memset.
559 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
560
561 /// KMSAN callback for task-local function argument shadow.
562 StructType *MsanContextStateTy;
563 FunctionCallee MsanGetContextStateFn;
564
565 /// Functions for poisoning/unpoisoning local variables
566 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
567
568 /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
569 /// pointers.
570 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
571 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
572 FunctionCallee MsanMetadataPtrForStore_1_8[4];
573 FunctionCallee MsanInstrumentAsmStoreFn;
574
575 /// Helper to choose between different MsanMetadataPtrXxx().
576 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
577
578 /// Memory map parameters used in application-to-shadow calculation.
579 const MemoryMapParams *MapParams;
580
581 /// Custom memory map parameters used when -msan-shadow-base or
582 // -msan-origin-base is provided.
583 MemoryMapParams CustomMapParams;
584
585 MDNode *ColdCallWeights;
586
587 /// Branch weights for origin store.
588 MDNode *OriginStoreWeights;
589
590 /// An empty volatile inline asm that prevents callback merge.
591 InlineAsm *EmptyAsm;
592};
593
594void insertModuleCtor(Module &M) {
595 getOrCreateSanitizerCtorAndInitFunctions(
596 M, kMsanModuleCtorName, kMsanInitName,
597 /*InitArgTypes=*/{},
598 /*InitArgs=*/{},
599 // This callback is invoked when the functions are created the first
600 // time. Hook them into the global ctors list in that case:
601 [&](Function *Ctor, FunctionCallee) {
602 if (!ClWithComdat) {
603 appendToGlobalCtors(M, Ctor, 0);
604 return;
605 }
606 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
607 Ctor->setComdat(MsanCtorComdat);
608 appendToGlobalCtors(M, Ctor, 0, Ctor);
609 });
610}
611
612/// A legacy function pass for msan instrumentation.
613///
614/// Instruments functions to detect unitialized reads.
615struct MemorySanitizerLegacyPass : public FunctionPass {
616 // Pass identification, replacement for typeid.
617 static char ID;
618
619 MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {})
620 : FunctionPass(ID), Options(Options) {}
621 StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; }
622
623 void getAnalysisUsage(AnalysisUsage &AU) const override {
624 AU.addRequired<TargetLibraryInfoWrapperPass>();
625 }
626
627 bool runOnFunction(Function &F) override {
628 return MSan->sanitizeFunction(
629 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F));
630 }
631 bool doInitialization(Module &M) override;
632
633 Optional<MemorySanitizer> MSan;
634 MemorySanitizerOptions Options;
635};
636
637template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
638 return (Opt.getNumOccurrences() > 0) ? Opt : Default;
639}
640
641} // end anonymous namespace
642
643MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K)
644 : Kernel(getOptOrDefault(ClEnableKmsan, K)),
645 TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
646 Recover(getOptOrDefault(ClKeepGoing, Kernel || R)) {}
647
648PreservedAnalyses MemorySanitizerPass::run(Function &F,
649 FunctionAnalysisManager &FAM) {
650 MemorySanitizer Msan(*F.getParent(), Options);
651 if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
652 return PreservedAnalyses::none();
653 return PreservedAnalyses::all();
654}
655
656PreservedAnalyses MemorySanitizerPass::run(Module &M,
657 ModuleAnalysisManager &AM) {
658 if (Options.Kernel)
659 return PreservedAnalyses::all();
660 insertModuleCtor(M);
661 return PreservedAnalyses::none();
662}
663
664char MemorySanitizerLegacyPass::ID = 0;
665
666INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
667 "MemorySanitizer: detects uninitialized reads.", false,static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
668 false)static void *initializeMemorySanitizerLegacyPassPassOnce(PassRegistry
&Registry) {
669INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
670INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
671 "MemorySanitizer: detects uninitialized reads.", false,PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
672 false)PassInfo *PI = new PassInfo( "MemorySanitizer: detects uninitialized reads."
, "msan", &MemorySanitizerLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<MemorySanitizerLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeMemorySanitizerLegacyPassPassFlag; void
llvm::initializeMemorySanitizerLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeMemorySanitizerLegacyPassPassFlag
, initializeMemorySanitizerLegacyPassPassOnce, std::ref(Registry
)); }
673
674FunctionPass *
675llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) {
676 return new MemorySanitizerLegacyPass(Options);
677}
678
679/// Create a non-const global initialized with the given string.
680///
681/// Creates a writable global for Str so that we can pass it to the
682/// run-time lib. Runtime uses first 4 bytes of the string to store the
683/// frame ID, so the string needs to be mutable.
684static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
685 StringRef Str) {
686 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
687 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
688 GlobalValue::PrivateLinkage, StrConst, "");
689}
690
691/// Create KMSAN API callbacks.
692void MemorySanitizer::createKernelApi(Module &M) {
693 IRBuilder<> IRB(*C);
694
695 // These will be initialized in insertKmsanPrologue().
696 RetvalTLS = nullptr;
697 RetvalOriginTLS = nullptr;
698 ParamTLS = nullptr;
699 ParamOriginTLS = nullptr;
700 VAArgTLS = nullptr;
701 VAArgOriginTLS = nullptr;
702 VAArgOverflowSizeTLS = nullptr;
703 // OriginTLS is unused in the kernel.
704 OriginTLS = nullptr;
705
706 // __msan_warning() in the kernel takes an origin.
707 WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
708 IRB.getInt32Ty());
709 // Requests the per-task context state (kmsan_context_state*) from the
710 // runtime library.
711 MsanContextStateTy = StructType::get(
712 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
713 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
714 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
715 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
716 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
717 OriginTy);
718 MsanGetContextStateFn = M.getOrInsertFunction(
719 "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
720
721 Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
722 PointerType::get(IRB.getInt32Ty(), 0));
723
724 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
725 std::string name_load =
726 "__msan_metadata_ptr_for_load_" + std::to_string(size);
727 std::string name_store =
728 "__msan_metadata_ptr_for_store_" + std::to_string(size);
729 MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
730 name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
731 MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
732 name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
733 }
734
735 MsanMetadataPtrForLoadN = M.getOrInsertFunction(
736 "__msan_metadata_ptr_for_load_n", RetTy,
737 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
738 MsanMetadataPtrForStoreN = M.getOrInsertFunction(
739 "__msan_metadata_ptr_for_store_n", RetTy,
740 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
741
742 // Functions for poisoning and unpoisoning memory.
743 MsanPoisonAllocaFn =
744 M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
745 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
746 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
747 "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
748}
749
750static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
751 return M.getOrInsertGlobal(Name, Ty, [&] {
752 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
753 nullptr, Name, nullptr,
754 GlobalVariable::InitialExecTLSModel);
755 });
756}
757
758/// Insert declarations for userspace-specific functions and globals.
759void MemorySanitizer::createUserspaceApi(Module &M) {
760 IRBuilder<> IRB(*C);
761 // Create the callback.
762 // FIXME: this function should have "Cold" calling conv,
763 // which is not yet implemented.
764 StringRef WarningFnName = Recover ? "__msan_warning"
765 : "__msan_warning_noreturn";
766 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
767
768 // Create the global TLS variables.
769 RetvalTLS =
770 getOrInsertGlobal(M, "__msan_retval_tls",
771 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
772
773 RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
774
775 ParamTLS =
776 getOrInsertGlobal(M, "__msan_param_tls",
777 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
778
779 ParamOriginTLS =
780 getOrInsertGlobal(M, "__msan_param_origin_tls",
781 ArrayType::get(OriginTy, kParamTLSSize / 4));
782
783 VAArgTLS =
784 getOrInsertGlobal(M, "__msan_va_arg_tls",
785 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
786
787 VAArgOriginTLS =
788 getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
789 ArrayType::get(OriginTy, kParamTLSSize / 4));
790
791 VAArgOverflowSizeTLS =
792 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
793 OriginTLS = getOrInsertGlobal(M, "__msan_origin_tls", IRB.getInt32Ty());
794
795 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
796 AccessSizeIndex++) {
797 unsigned AccessSize = 1 << AccessSizeIndex;
798 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
799 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
800 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
801 IRB.getInt32Ty());
802
803 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
804 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
805 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
806 IRB.getInt8PtrTy(), IRB.getInt32Ty());
807 }
808
809 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
810 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
811 IRB.getInt8PtrTy(), IntptrTy);
812 MsanPoisonStackFn =
813 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
814 IRB.getInt8PtrTy(), IntptrTy);
815}
816
817/// Insert extern declaration of runtime-provided functions and globals.
818void MemorySanitizer::initializeCallbacks(Module &M) {
819 // Only do this once.
820 if (CallbacksInitialized)
821 return;
822
823 IRBuilder<> IRB(*C);
824 // Initialize callbacks that are common for kernel and userspace
825 // instrumentation.
826 MsanChainOriginFn = M.getOrInsertFunction(
827 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
828 MemmoveFn = M.getOrInsertFunction(
829 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
830 IRB.getInt8PtrTy(), IntptrTy);
831 MemcpyFn = M.getOrInsertFunction(
832 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
833 IntptrTy);
834 MemsetFn = M.getOrInsertFunction(
835 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
836 IntptrTy);
837 // We insert an empty inline asm after __msan_report* to avoid callback merge.
838 EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
839 StringRef(""), StringRef(""),
840 /*hasSideEffects=*/true);
841
842 MsanInstrumentAsmStoreFn =
843 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
844 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
845
846 if (CompileKernel) {
847 createKernelApi(M);
848 } else {
849 createUserspaceApi(M);
850 }
851 CallbacksInitialized = true;
852}
853
854FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
855 int size) {
856 FunctionCallee *Fns =
857 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
858 switch (size) {
859 case 1:
860 return Fns[0];
861 case 2:
862 return Fns[1];
863 case 4:
864 return Fns[2];
865 case 8:
866 return Fns[3];
867 default:
868 return nullptr;
869 }
870}
871
872/// Module-level initialization.
873///
874/// inserts a call to __msan_init to the module's constructor list.
875void MemorySanitizer::initializeModule(Module &M) {
876 auto &DL = M.getDataLayout();
877
878 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
879 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
880 // Check the overrides first
881 if (ShadowPassed || OriginPassed) {
882 CustomMapParams.AndMask = ClAndMask;
883 CustomMapParams.XorMask = ClXorMask;
884 CustomMapParams.ShadowBase = ClShadowBase;
885 CustomMapParams.OriginBase = ClOriginBase;
886 MapParams = &CustomMapParams;
887 } else {
888 Triple TargetTriple(M.getTargetTriple());
889 switch (TargetTriple.getOS()) {
890 case Triple::FreeBSD:
891 switch (TargetTriple.getArch()) {
892 case Triple::x86_64:
893 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
894 break;
895 case Triple::x86:
896 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
897 break;
898 default:
899 report_fatal_error("unsupported architecture");
900 }
901 break;
902 case Triple::NetBSD:
903 switch (TargetTriple.getArch()) {
904 case Triple::x86_64:
905 MapParams = NetBSD_X86_MemoryMapParams.bits64;
906 break;
907 default:
908 report_fatal_error("unsupported architecture");
909 }
910 break;
911 case Triple::Linux:
912 switch (TargetTriple.getArch()) {
913 case Triple::x86_64:
914 MapParams = Linux_X86_MemoryMapParams.bits64;
915 break;
916 case Triple::x86:
917 MapParams = Linux_X86_MemoryMapParams.bits32;
918 break;
919 case Triple::mips64:
920 case Triple::mips64el:
921 MapParams = Linux_MIPS_MemoryMapParams.bits64;
922 break;
923 case Triple::ppc64:
924 case Triple::ppc64le:
925 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
926 break;
927 case Triple::aarch64:
928 case Triple::aarch64_be:
929 MapParams = Linux_ARM_MemoryMapParams.bits64;
930 break;
931 default:
932 report_fatal_error("unsupported architecture");
933 }
934 break;
935 default:
936 report_fatal_error("unsupported operating system");
937 }
938 }
939
940 C = &(M.getContext());
941 IRBuilder<> IRB(*C);
942 IntptrTy = IRB.getIntPtrTy(DL);
943 OriginTy = IRB.getInt32Ty();
944
945 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
946 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
947
948 if (!CompileKernel) {
949 if (TrackOrigins)
950 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
951 return new GlobalVariable(
952 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
953 IRB.getInt32(TrackOrigins), "__msan_track_origins");
954 });
955
956 if (Recover)
957 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
958 return new GlobalVariable(M, IRB.getInt32Ty(), true,
959 GlobalValue::WeakODRLinkage,
960 IRB.getInt32(Recover), "__msan_keep_going");
961 });
962}
963}
964
965bool MemorySanitizerLegacyPass::doInitialization(Module &M) {
966 if (!Options.Kernel)
967 insertModuleCtor(M);
968 MSan.emplace(M, Options);
969 return true;
970}
971
972namespace {
973
974/// A helper class that handles instrumentation of VarArg
975/// functions on a particular platform.
976///
977/// Implementations are expected to insert the instrumentation
978/// necessary to propagate argument shadow through VarArg function
979/// calls. Visit* methods are called during an InstVisitor pass over
980/// the function, and should avoid creating new basic blocks. A new
981/// instance of this class is created for each instrumented function.
982struct VarArgHelper {
983 virtual ~VarArgHelper() = default;
984
985 /// Visit a CallSite.
986 virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
987
988 /// Visit a va_start call.
989 virtual void visitVAStartInst(VAStartInst &I) = 0;
990
991 /// Visit a va_copy call.
992 virtual void visitVACopyInst(VACopyInst &I) = 0;
993
994 /// Finalize function instrumentation.
995 ///
996 /// This method is called after visiting all interesting (see above)
997 /// instructions in a function.
998 virtual void finalizeInstrumentation() = 0;
999};
1000
1001struct MemorySanitizerVisitor;
1002
1003} // end anonymous namespace
1004
1005static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1006 MemorySanitizerVisitor &Visitor);
1007
1008static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1009 if (TypeSize <= 8) return 0;
1010 return Log2_32_Ceil((TypeSize + 7) / 8);
1011}
1012
1013namespace {
1014
1015/// This class does all the work for a given function. Store and Load
1016/// instructions store and load corresponding shadow and origin
1017/// values. Most instructions propagate shadow from arguments to their
1018/// return values. Certain instructions (most importantly, BranchInst)
1019/// test their argument shadow and print reports (with a runtime call) if it's
1020/// non-zero.
1021struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1022 Function &F;
1023 MemorySanitizer &MS;
1024 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1025 ValueMap<Value*, Value*> ShadowMap, OriginMap;
1026 std::unique_ptr<VarArgHelper> VAHelper;
1027 const TargetLibraryInfo *TLI;
1028 BasicBlock *ActualFnStart;
1029
1030 // The following flags disable parts of MSan instrumentation based on
1031 // blacklist contents and command-line options.
1032 bool InsertChecks;
1033 bool PropagateShadow;
1034 bool PoisonStack;
1035 bool PoisonUndef;
1036 bool CheckReturnValue;
1037
1038 struct ShadowOriginAndInsertPoint {
1039 Value *Shadow;
1040 Value *Origin;
1041 Instruction *OrigIns;
1042
1043 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1044 : Shadow(S), Origin(O), OrigIns(I) {}
1045 };
1046 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1047 bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1048 SmallSet<AllocaInst *, 16> AllocaSet;
1049 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1050 SmallVector<StoreInst *, 16> StoreList;
1051
1052 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1053 const TargetLibraryInfo &TLI)
1054 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1055 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
1056 InsertChecks = SanitizeFunction;
1057 PropagateShadow = SanitizeFunction;
1058 PoisonStack = SanitizeFunction && ClPoisonStack;
1059 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1060 // FIXME: Consider using SpecialCaseList to specify a list of functions that
1061 // must always return fully initialized values. For now, we hardcode "main".
1062 CheckReturnValue = SanitizeFunction && (F.getName() == "main");
1063
1064 MS.initializeCallbacks(*F.getParent());
1065 if (MS.CompileKernel)
1066 ActualFnStart = insertKmsanPrologue(F);
1067 else
1068 ActualFnStart = &F.getEntryBlock();
1069
1070 LLVM_DEBUG(if (!InsertChecks) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1071 << "MemorySanitizer is not inserting checks into '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1072 << F.getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
;
1073 }
1074
1075 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1076 if (MS.TrackOrigins <= 1) return V;
1077 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1078 }
1079
1080 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1081 const DataLayout &DL = F.getParent()->getDataLayout();
1082 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1083 if (IntptrSize == kOriginSize) return Origin;
1084 assert(IntptrSize == kOriginSize * 2)((IntptrSize == kOriginSize * 2) ? static_cast<void> (0
) : __assert_fail ("IntptrSize == kOriginSize * 2", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1084, __PRETTY_FUNCTION__))
;
1085 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1086 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1087 }
1088
1089 /// Fill memory range with the given origin value.
1090 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1091 unsigned Size, Align Alignment) {
1092 const DataLayout &DL = F.getParent()->getDataLayout();
1093 const Align IntptrAlignment = Align(DL.getABITypeAlignment(MS.IntptrTy));
1094 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1095 assert(IntptrAlignment >= kMinOriginAlignment)((IntptrAlignment >= kMinOriginAlignment) ? static_cast<
void> (0) : __assert_fail ("IntptrAlignment >= kMinOriginAlignment"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1095, __PRETTY_FUNCTION__))
;
1096 assert(IntptrSize >= kOriginSize)((IntptrSize >= kOriginSize) ? static_cast<void> (0)
: __assert_fail ("IntptrSize >= kOriginSize", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1096, __PRETTY_FUNCTION__))
;
1097
1098 unsigned Ofs = 0;
1099 Align CurrentAlignment = Alignment;
1100 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1101 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1102 Value *IntptrOriginPtr =
1103 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1104 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1105 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1106 : IntptrOriginPtr;
1107 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment.value());
1108 Ofs += IntptrSize / kOriginSize;
1109 CurrentAlignment = IntptrAlignment;
1110 }
1111 }
1112
1113 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1114 Value *GEP =
1115 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1116 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment.value());
1117 CurrentAlignment = kMinOriginAlignment;
1118 }
1119 }
1120
1121 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1122 Value *OriginPtr, Align Alignment, bool AsCall) {
1123 const DataLayout &DL = F.getParent()->getDataLayout();
1124 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1125 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1126 if (Shadow->getType()->isAggregateType()) {
1
Calling 'Type::isAggregateType'
5
Returning from 'Type::isAggregateType'
6
Taking false branch
1127 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1128 OriginAlignment);
1129 } else {
1130 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
7
Calling 'MemorySanitizerVisitor::convertToShadowTyNoVec'
21
Returning from 'MemorySanitizerVisitor::convertToShadowTyNoVec'
22
'ConvertedShadow' initialized here
1131 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
23
Assuming null pointer is passed into cast
24
Assuming pointer value is null
1132 if (ConstantShadow
24.1
'ConstantShadow' is null
24.1
'ConstantShadow' is null
24.1
'ConstantShadow' is null
) {
25
Taking false branch
1133 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
1134 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1135 OriginAlignment);
1136 return;
1137 }
1138
1139 unsigned TypeSizeInBits =
1140 DL.getTypeSizeInBits(ConvertedShadow->getType());
26
Called C++ object pointer is null
1141 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1142 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1143 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1144 Value *ConvertedShadow2 = IRB.CreateZExt(
1145 ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1146 IRB.CreateCall(Fn, {ConvertedShadow2,
1147 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
1148 Origin});
1149 } else {
1150 Value *Cmp = IRB.CreateICmpNE(
1151 ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
1152 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1153 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1154 IRBuilder<> IRBNew(CheckTerm);
1155 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1156 OriginAlignment);
1157 }
1158 }
1159 }
1160
1161 void materializeStores(bool InstrumentWithCalls) {
1162 for (StoreInst *SI : StoreList) {
1163 IRBuilder<> IRB(SI);
1164 Value *Val = SI->getValueOperand();
1165 Value *Addr = SI->getPointerOperand();
1166 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1167 Value *ShadowPtr, *OriginPtr;
1168 Type *ShadowTy = Shadow->getType();
1169 const Align Alignment = assumeAligned(SI->getAlignment());
1170 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1171 std::tie(ShadowPtr, OriginPtr) =
1172 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1173
1174 StoreInst *NewSI =
1175 IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment.value());
1176 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " STORE: " << *NewSI <<
"\n"; } } while (false)
;
1177 (void)NewSI;
1178
1179 if (SI->isAtomic())
1180 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1181
1182 if (MS.TrackOrigins && !SI->isAtomic())
1183 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1184 OriginAlignment, InstrumentWithCalls);
1185 }
1186 }
1187
1188 /// Helper function to insert a warning at IRB's current insert point.
1189 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1190 if (!Origin)
1191 Origin = (Value *)IRB.getInt32(0);
1192 if (MS.CompileKernel) {
1193 IRB.CreateCall(MS.WarningFn, Origin);
1194 } else {
1195 if (MS.TrackOrigins) {
1196 IRB.CreateStore(Origin, MS.OriginTLS);
1197 }
1198 IRB.CreateCall(MS.WarningFn, {});
1199 }
1200 IRB.CreateCall(MS.EmptyAsm, {});
1201 // FIXME: Insert UnreachableInst if !MS.Recover?
1202 // This may invalidate some of the following checks and needs to be done
1203 // at the very end.
1204 }
1205
1206 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
1207 bool AsCall) {
1208 IRBuilder<> IRB(OrigIns);
1209 LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD0 : " << *Shadow <<
"\n"; } } while (false)
;
1210 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
1211 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD1 : " << *ConvertedShadow
<< "\n"; } } while (false)
;
1212
1213 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
1214 if (ConstantShadow) {
1215 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
1216 insertWarningFn(IRB, Origin);
1217 }
1218 return;
1219 }
1220
1221 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
1222
1223 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1224 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1225 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1226 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1227 Value *ConvertedShadow2 =
1228 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1229 IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
1230 ? Origin
1231 : (Value *)IRB.getInt32(0)});
1232 } else {
1233 Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
1234 getCleanShadow(ConvertedShadow), "_mscmp");
1235 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1236 Cmp, OrigIns,
1237 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1238
1239 IRB.SetInsertPoint(CheckTerm);
1240 insertWarningFn(IRB, Origin);
1241 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CHECK: " << *Cmp <<
"\n"; } } while (false)
;
1242 }
1243 }
1244
1245 void materializeChecks(bool InstrumentWithCalls) {
1246 for (const auto &ShadowData : InstrumentationList) {
1247 Instruction *OrigIns = ShadowData.OrigIns;
1248 Value *Shadow = ShadowData.Shadow;
1249 Value *Origin = ShadowData.Origin;
1250 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
1251 }
1252 LLVM_DEBUG(dbgs() << "DONE:\n" << F)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DONE:\n" << F; } } while (
false)
;
1253 }
1254
1255 BasicBlock *insertKmsanPrologue(Function &F) {
1256 BasicBlock *ret =
1257 SplitBlock(&F.getEntryBlock(), F.getEntryBlock().getFirstNonPHI());
1258 IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
1259 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1260 Constant *Zero = IRB.getInt32(0);
1261 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1262 {Zero, IRB.getInt32(0)}, "param_shadow");
1263 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1264 {Zero, IRB.getInt32(1)}, "retval_shadow");
1265 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1266 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1267 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1268 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1269 MS.VAArgOverflowSizeTLS =
1270 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1271 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1272 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1273 {Zero, IRB.getInt32(5)}, "param_origin");
1274 MS.RetvalOriginTLS =
1275 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1276 {Zero, IRB.getInt32(6)}, "retval_origin");
1277 return ret;
1278 }
1279
1280 /// Add MemorySanitizer instrumentation to a function.
1281 bool runOnFunction() {
1282 // In the presence of unreachable blocks, we may see Phi nodes with
1283 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1284 // blocks, such nodes will not have any shadow value associated with them.
1285 // It's easier to remove unreachable blocks than deal with missing shadow.
1286 removeUnreachableBlocks(F);
1287
1288 // Iterate all BBs in depth-first order and create shadow instructions
1289 // for all instructions (where applicable).
1290 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1291 for (BasicBlock *BB : depth_first(ActualFnStart))
1292 visit(*BB);
1293
1294 // Finalize PHI nodes.
1295 for (PHINode *PN : ShadowPHINodes) {
1296 PHINode *PNS = cast<PHINode>(getShadow(PN));
1297 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1298 size_t NumValues = PN->getNumIncomingValues();
1299 for (size_t v = 0; v < NumValues; v++) {
1300 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1301 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1302 }
1303 }
1304
1305 VAHelper->finalizeInstrumentation();
1306
1307 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1308 // instrumenting only allocas.
1309 if (InstrumentLifetimeStart) {
1310 for (auto Item : LifetimeStartList) {
1311 instrumentAlloca(*Item.second, Item.first);
1312 AllocaSet.erase(Item.second);
1313 }
1314 }
1315 // Poison the allocas for which we didn't instrument the corresponding
1316 // lifetime intrinsics.
1317 for (AllocaInst *AI : AllocaSet)
1318 instrumentAlloca(*AI);
1319
1320 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1321 InstrumentationList.size() + StoreList.size() >
1322 (unsigned)ClInstrumentationWithCallThreshold;
1323
1324 // Insert shadow value checks.
1325 materializeChecks(InstrumentWithCalls);
1326
1327 // Delayed instrumentation of StoreInst.
1328 // This may not add new address checks.
1329 materializeStores(InstrumentWithCalls);
1330
1331 return true;
1332 }
1333
1334 /// Compute the shadow type that corresponds to a given Value.
1335 Type *getShadowTy(Value *V) {
1336 return getShadowTy(V->getType());
1337 }
1338
1339 /// Compute the shadow type that corresponds to a given Type.
1340 Type *getShadowTy(Type *OrigTy) {
1341 if (!OrigTy->isSized()) {
1342 return nullptr;
1343 }
1344 // For integer type, shadow is the same as the original type.
1345 // This may return weird-sized types like i1.
1346 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1347 return IT;
1348 const DataLayout &DL = F.getParent()->getDataLayout();
1349 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1350 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1351 return VectorType::get(IntegerType::get(*MS.C, EltSize),
1352 VT->getNumElements());
1353 }
1354 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1355 return ArrayType::get(getShadowTy(AT->getElementType()),
1356 AT->getNumElements());
1357 }
1358 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1359 SmallVector<Type*, 4> Elements;
1360 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1361 Elements.push_back(getShadowTy(ST->getElementType(i)));
1362 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1363 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "getShadowTy: " << *ST <<
" ===> " << *Res << "\n"; } } while (false)
;
1364 return Res;
1365 }
1366 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1367 return IntegerType::get(*MS.C, TypeSize);
1368 }
1369
1370 /// Flatten a vector type.
1371 Type *getShadowTyNoVec(Type *ty) {
1372 if (VectorType *vt = dyn_cast<VectorType>(ty))
1373 return IntegerType::get(*MS.C, vt->getBitWidth());
1374 return ty;
1375 }
1376
1377 /// Convert a shadow value to it's flattened variant.
1378 Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
1379 Type *Ty = V->getType();
1380 Type *NoVecTy = getShadowTyNoVec(Ty);
1381 if (Ty == NoVecTy) return V;
8
Assuming 'Ty' is not equal to 'NoVecTy'
9
Taking false branch
1382 return IRB.CreateBitCast(V, NoVecTy);
10
Calling 'IRBuilder::CreateBitCast'
19
Returning from 'IRBuilder::CreateBitCast'
20
Returning pointer
1383 }
1384
1385 /// Compute the integer shadow offset that corresponds to a given
1386 /// application address.
1387 ///
1388 /// Offset = (Addr & ~AndMask) ^ XorMask
1389 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1390 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1391
1392 uint64_t AndMask = MS.MapParams->AndMask;
1393 if (AndMask)
1394 OffsetLong =
1395 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1396
1397 uint64_t XorMask = MS.MapParams->XorMask;
1398 if (XorMask)
1399 OffsetLong =
1400 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1401 return OffsetLong;
1402 }
1403
1404 /// Compute the shadow and origin addresses corresponding to a given
1405 /// application address.
1406 ///
1407 /// Shadow = ShadowBase + Offset
1408 /// Origin = (OriginBase + Offset) & ~3ULL
1409 std::pair<Value *, Value *>
1410 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1411 MaybeAlign Alignment) {
1412 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1413 Value *ShadowLong = ShadowOffset;
1414 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1415 if (ShadowBase != 0) {
1416 ShadowLong =
1417 IRB.CreateAdd(ShadowLong,
1418 ConstantInt::get(MS.IntptrTy, ShadowBase));
1419 }
1420 Value *ShadowPtr =
1421 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1422 Value *OriginPtr = nullptr;
1423 if (MS.TrackOrigins) {
1424 Value *OriginLong = ShadowOffset;
1425 uint64_t OriginBase = MS.MapParams->OriginBase;
1426 if (OriginBase != 0)
1427 OriginLong = IRB.CreateAdd(OriginLong,
1428 ConstantInt::get(MS.IntptrTy, OriginBase));
1429 if (!Alignment || *Alignment < kMinOriginAlignment) {
1430 uint64_t Mask = kMinOriginAlignment.value() - 1;
1431 OriginLong =
1432 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1433 }
1434 OriginPtr =
1435 IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1436 }
1437 return std::make_pair(ShadowPtr, OriginPtr);
1438 }
1439
1440 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1441 IRBuilder<> &IRB,
1442 Type *ShadowTy,
1443 bool isStore) {
1444 Value *ShadowOriginPtrs;
1445 const DataLayout &DL = F.getParent()->getDataLayout();
1446 int Size = DL.getTypeStoreSize(ShadowTy);
1447
1448 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1449 Value *AddrCast =
1450 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
1451 if (Getter) {
1452 ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1453 } else {
1454 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1455 ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1456 : MS.MsanMetadataPtrForLoadN,
1457 {AddrCast, SizeVal});
1458 }
1459 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1460 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1461 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1462
1463 return std::make_pair(ShadowPtr, OriginPtr);
1464 }
1465
1466 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1467 Type *ShadowTy,
1468 MaybeAlign Alignment,
1469 bool isStore) {
1470 if (MS.CompileKernel)
1471 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1472 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1473 }
1474
1475 /// Compute the shadow address for a given function argument.
1476 ///
1477 /// Shadow = ParamTLS+ArgOffset.
1478 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1479 int ArgOffset) {
1480 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1481 if (ArgOffset)
1482 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1483 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1484 "_msarg");
1485 }
1486
1487 /// Compute the origin address for a given function argument.
1488 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1489 int ArgOffset) {
1490 if (!MS.TrackOrigins)
1491 return nullptr;
1492 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1493 if (ArgOffset)
1494 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1495 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1496 "_msarg_o");
1497 }
1498
1499 /// Compute the shadow address for a retval.
1500 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1501 return IRB.CreatePointerCast(MS.RetvalTLS,
1502 PointerType::get(getShadowTy(A), 0),
1503 "_msret");
1504 }
1505
1506 /// Compute the origin address for a retval.
1507 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1508 // We keep a single origin for the entire retval. Might be too optimistic.
1509 return MS.RetvalOriginTLS;
1510 }
1511
1512 /// Set SV to be the shadow value for V.
1513 void setShadow(Value *V, Value *SV) {
1514 assert(!ShadowMap.count(V) && "Values may only have one shadow")((!ShadowMap.count(V) && "Values may only have one shadow"
) ? static_cast<void> (0) : __assert_fail ("!ShadowMap.count(V) && \"Values may only have one shadow\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1514, __PRETTY_FUNCTION__))
;
1515 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1516 }
1517
1518 /// Set Origin to be the origin value for V.
1519 void setOrigin(Value *V, Value *Origin) {
1520 if (!MS.TrackOrigins) return;
1521 assert(!OriginMap.count(V) && "Values may only have one origin")((!OriginMap.count(V) && "Values may only have one origin"
) ? static_cast<void> (0) : __assert_fail ("!OriginMap.count(V) && \"Values may only have one origin\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1521, __PRETTY_FUNCTION__))
;
1522 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ORIGIN: " << *V << " ==> "
<< *Origin << "\n"; } } while (false)
;
1523 OriginMap[V] = Origin;
1524 }
1525
1526 Constant *getCleanShadow(Type *OrigTy) {
1527 Type *ShadowTy = getShadowTy(OrigTy);
1528 if (!ShadowTy)
1529 return nullptr;
1530 return Constant::getNullValue(ShadowTy);
1531 }
1532
1533 /// Create a clean shadow value for a given value.
1534 ///
1535 /// Clean shadow (all zeroes) means all bits of the value are defined
1536 /// (initialized).
1537 Constant *getCleanShadow(Value *V) {
1538 return getCleanShadow(V->getType());
1539 }
1540
1541 /// Create a dirty shadow of a given shadow type.
1542 Constant *getPoisonedShadow(Type *ShadowTy) {
1543 assert(ShadowTy)((ShadowTy) ? static_cast<void> (0) : __assert_fail ("ShadowTy"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1543, __PRETTY_FUNCTION__))
;
1544 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1545 return Constant::getAllOnesValue(ShadowTy);
1546 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1547 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1548 getPoisonedShadow(AT->getElementType()));
1549 return ConstantArray::get(AT, Vals);
1550 }
1551 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1552 SmallVector<Constant *, 4> Vals;
1553 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1554 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1555 return ConstantStruct::get(ST, Vals);
1556 }
1557 llvm_unreachable("Unexpected shadow type")::llvm::llvm_unreachable_internal("Unexpected shadow type", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1557)
;
1558 }
1559
1560 /// Create a dirty shadow for a given value.
1561 Constant *getPoisonedShadow(Value *V) {
1562 Type *ShadowTy = getShadowTy(V);
1563 if (!ShadowTy)
1564 return nullptr;
1565 return getPoisonedShadow(ShadowTy);
1566 }
1567
1568 /// Create a clean (zero) origin.
1569 Value *getCleanOrigin() {
1570 return Constant::getNullValue(MS.OriginTy);
1571 }
1572
1573 /// Get the shadow value for a given Value.
1574 ///
1575 /// This function either returns the value set earlier with setShadow,
1576 /// or extracts if from ParamTLS (for function arguments).
1577 Value *getShadow(Value *V) {
1578 if (!PropagateShadow) return getCleanShadow(V);
1579 if (Instruction *I = dyn_cast<Instruction>(V)) {
1580 if (I->getMetadata("nosanitize"))
1581 return getCleanShadow(V);
1582 // For instructions the shadow is already stored in the map.
1583 Value *Shadow = ShadowMap[V];
1584 if (!Shadow) {
1585 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "No shadow: " << *V <<
"\n" << *(I->getParent()); } } while (false)
;
1586 (void)I;
1587 assert(Shadow && "No shadow for a value")((Shadow && "No shadow for a value") ? static_cast<
void> (0) : __assert_fail ("Shadow && \"No shadow for a value\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1587, __PRETTY_FUNCTION__))
;
1588 }
1589 return Shadow;
1590 }
1591 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1592 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1593 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Undef: " << *U << " ==> "
<< *AllOnes << "\n"; } } while (false)
;
1594 (void)U;
1595 return AllOnes;
1596 }
1597 if (Argument *A = dyn_cast<Argument>(V)) {
1598 // For arguments we compute the shadow on demand and store it in the map.
1599 Value **ShadowPtr = &ShadowMap[V];
1600 if (*ShadowPtr)
1601 return *ShadowPtr;
1602 Function *F = A->getParent();
1603 IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
1604 unsigned ArgOffset = 0;
1605 const DataLayout &DL = F->getParent()->getDataLayout();
1606 for (auto &FArg : F->args()) {
1607 if (!FArg.getType()->isSized()) {
1608 LLVM_DEBUG(dbgs() << "Arg is not sized\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg is not sized\n"; } } while (
false)
;
1609 continue;
1610 }
1611 unsigned Size =
1612 FArg.hasByValAttr()
1613 ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1614 : DL.getTypeAllocSize(FArg.getType());
1615 if (A == &FArg) {
1616 bool Overflow = ArgOffset + Size > kParamTLSSize;
1617 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1618 if (FArg.hasByValAttr()) {
1619 // ByVal pointer itself has clean shadow. We copy the actual
1620 // argument shadow to the underlying memory.
1621 // Figure out maximal valid memcpy alignment.
1622 const Align ArgAlign = DL.getValueOrABITypeAlignment(
1623 MaybeAlign(FArg.getParamAlignment()),
1624 A->getType()->getPointerElementType());
1625 Value *CpShadowPtr =
1626 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1627 /*isStore*/ true)
1628 .first;
1629 // TODO(glider): need to copy origins.
1630 if (Overflow) {
1631 // ParamTLS overflow.
1632 EntryIRB.CreateMemSet(
1633 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1634 Size, ArgAlign);
1635 } else {
1636 const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1637 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1638 CopyAlign, Size);
1639 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ByValCpy: " << *Cpy <<
"\n"; } } while (false)
;
1640 (void)Cpy;
1641 }
1642 *ShadowPtr = getCleanShadow(V);
1643 } else {
1644 if (Overflow) {
1645 // ParamTLS overflow.
1646 *ShadowPtr = getCleanShadow(V);
1647 } else {
1648 *ShadowPtr = EntryIRB.CreateAlignedLoad(
1649 getShadowTy(&FArg), Base, kShadowTLSAlignment.value());
1650 }
1651 }
1652 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << **ShadowPtr << "\n"; } } while (false
)
1653 << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << **ShadowPtr << "\n"; } } while (false
)
;
1654 if (MS.TrackOrigins && !Overflow) {
1655 Value *OriginPtr =
1656 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1657 setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1658 } else {
1659 setOrigin(A, getCleanOrigin());
1660 }
1661 }
1662 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1663 }
1664 assert(*ShadowPtr && "Could not find shadow for an argument")((*ShadowPtr && "Could not find shadow for an argument"
) ? static_cast<void> (0) : __assert_fail ("*ShadowPtr && \"Could not find shadow for an argument\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1664, __PRETTY_FUNCTION__))
;
1665 return *ShadowPtr;
1666 }
1667 // For everything else the shadow is zero.
1668 return getCleanShadow(V);
1669 }
1670
1671 /// Get the shadow for i-th argument of the instruction I.
1672 Value *getShadow(Instruction *I, int i) {
1673 return getShadow(I->getOperand(i));
1674 }
1675
1676 /// Get the origin for a value.
1677 Value *getOrigin(Value *V) {
1678 if (!MS.TrackOrigins) return nullptr;
1679 if (!PropagateShadow) return getCleanOrigin();
1680 if (isa<Constant>(V)) return getCleanOrigin();
1681 assert((isa<Instruction>(V) || isa<Argument>(V)) &&(((isa<Instruction>(V) || isa<Argument>(V)) &&
"Unexpected value type in getOrigin()") ? static_cast<void
> (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1682, __PRETTY_FUNCTION__))
1682 "Unexpected value type in getOrigin()")(((isa<Instruction>(V) || isa<Argument>(V)) &&
"Unexpected value type in getOrigin()") ? static_cast<void
> (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1682, __PRETTY_FUNCTION__))
;
1683 if (Instruction *I = dyn_cast<Instruction>(V)) {
1684 if (I->getMetadata("nosanitize"))
1685 return getCleanOrigin();
1686 }
1687 Value *Origin = OriginMap[V];
1688 assert(Origin && "Missing origin")((Origin && "Missing origin") ? static_cast<void>
(0) : __assert_fail ("Origin && \"Missing origin\"",
"/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1688, __PRETTY_FUNCTION__))
;
1689 return Origin;
1690 }
1691
1692 /// Get the origin for i-th argument of the instruction I.
1693 Value *getOrigin(Instruction *I, int i) {
1694 return getOrigin(I->getOperand(i));
1695 }
1696
1697 /// Remember the place where a shadow check should be inserted.
1698 ///
1699 /// This location will be later instrumented with a check that will print a
1700 /// UMR warning in runtime if the shadow value is not 0.
1701 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1702 assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1702, __PRETTY_FUNCTION__))
;
1703 if (!InsertChecks) return;
1704#ifndef NDEBUG
1705 Type *ShadowTy = Shadow->getType();
1706 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&(((isa<IntegerType>(ShadowTy) || isa<VectorType>(
ShadowTy)) && "Can only insert checks for integer and vector shadow types"
) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1707, __PRETTY_FUNCTION__))
1707 "Can only insert checks for integer and vector shadow types")(((isa<IntegerType>(ShadowTy) || isa<VectorType>(
ShadowTy)) && "Can only insert checks for integer and vector shadow types"
) ? static_cast<void> (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) && \"Can only insert checks for integer and vector shadow types\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1707, __PRETTY_FUNCTION__))
;
1708#endif
1709 InstrumentationList.push_back(
1710 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1711 }
1712
1713 /// Remember the place where a shadow check should be inserted.
1714 ///
1715 /// This location will be later instrumented with a check that will print a
1716 /// UMR warning in runtime if the value is not fully defined.
1717 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1718 assert(Val)((Val) ? static_cast<void> (0) : __assert_fail ("Val", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1718, __PRETTY_FUNCTION__))
;
1719 Value *Shadow, *Origin;
1720 if (ClCheckConstantShadow) {
1721 Shadow = getShadow(Val);
1722 if (!Shadow) return;
1723 Origin = getOrigin(Val);
1724 } else {
1725 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1726 if (!Shadow) return;
1727 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1728 }
1729 insertShadowCheck(Shadow, Origin, OrigIns);
1730 }
1731
1732 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1733 switch (a) {
1734 case AtomicOrdering::NotAtomic:
1735 return AtomicOrdering::NotAtomic;
1736 case AtomicOrdering::Unordered:
1737 case AtomicOrdering::Monotonic:
1738 case AtomicOrdering::Release:
1739 return AtomicOrdering::Release;
1740 case AtomicOrdering::Acquire:
1741 case AtomicOrdering::AcquireRelease:
1742 return AtomicOrdering::AcquireRelease;
1743 case AtomicOrdering::SequentiallyConsistent:
1744 return AtomicOrdering::SequentiallyConsistent;
1745 }
1746 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1746)
;
1747 }
1748
1749 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1750 switch (a) {
1751 case AtomicOrdering::NotAtomic:
1752 return AtomicOrdering::NotAtomic;
1753 case AtomicOrdering::Unordered:
1754 case AtomicOrdering::Monotonic:
1755 case AtomicOrdering::Acquire:
1756 return AtomicOrdering::Acquire;
1757 case AtomicOrdering::Release:
1758 case AtomicOrdering::AcquireRelease:
1759 return AtomicOrdering::AcquireRelease;
1760 case AtomicOrdering::SequentiallyConsistent:
1761 return AtomicOrdering::SequentiallyConsistent;
1762 }
1763 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1763)
;
1764 }
1765
1766 // ------------------- Visitors.
1767 using InstVisitor<MemorySanitizerVisitor>::visit;
1768 void visit(Instruction &I) {
1769 if (!I.getMetadata("nosanitize"))
1770 InstVisitor<MemorySanitizerVisitor>::visit(I);
1771 }
1772
1773 /// Instrument LoadInst
1774 ///
1775 /// Loads the corresponding shadow and (optionally) origin.
1776 /// Optionally, checks that the load address is fully defined.
1777 void visitLoadInst(LoadInst &I) {
1778 assert(I.getType()->isSized() && "Load type must have size")((I.getType()->isSized() && "Load type must have size"
) ? static_cast<void> (0) : __assert_fail ("I.getType()->isSized() && \"Load type must have size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1778, __PRETTY_FUNCTION__))
;
1779 assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0)
: __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1779, __PRETTY_FUNCTION__))
;
1780 IRBuilder<> IRB(I.getNextNode());
1781 Type *ShadowTy = getShadowTy(&I);
1782 Value *Addr = I.getPointerOperand();
1783 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
1784 const Align Alignment = assumeAligned(I.getAlignment());
1785 if (PropagateShadow) {
1786 std::tie(ShadowPtr, OriginPtr) =
1787 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1788 setShadow(&I, IRB.CreateAlignedLoad(ShadowTy, ShadowPtr,
1789 Alignment.value(), "_msld"));
1790 } else {
1791 setShadow(&I, getCleanShadow(&I));
1792 }
1793
1794 if (ClCheckAccessAddress)
1795 insertShadowCheck(I.getPointerOperand(), &I);
1796
1797 if (I.isAtomic())
1798 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1799
1800 if (MS.TrackOrigins) {
1801 if (PropagateShadow) {
1802 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1803 setOrigin(&I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr,
1804 OriginAlignment.value()));
1805 } else {
1806 setOrigin(&I, getCleanOrigin());
1807 }
1808 }
1809 }
1810
1811 /// Instrument StoreInst
1812 ///
1813 /// Stores the corresponding shadow and (optionally) origin.
1814 /// Optionally, checks that the store address is fully defined.
1815 void visitStoreInst(StoreInst &I) {
1816 StoreList.push_back(&I);
1817 if (ClCheckAccessAddress)
1818 insertShadowCheck(I.getPointerOperand(), &I);
1819 }
1820
1821 void handleCASOrRMW(Instruction &I) {
1822 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I))((isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1822, __PRETTY_FUNCTION__))
;
1823
1824 IRBuilder<> IRB(&I);
1825 Value *Addr = I.getOperand(0);
1826 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(), Align::None(),
1827 /*isStore*/ true)
1828 .first;
1829
1830 if (ClCheckAccessAddress)
1831 insertShadowCheck(Addr, &I);
1832
1833 // Only test the conditional argument of cmpxchg instruction.
1834 // The other argument can potentially be uninitialized, but we can not
1835 // detect this situation reliably without possible false positives.
1836 if (isa<AtomicCmpXchgInst>(I))
1837 insertShadowCheck(I.getOperand(1), &I);
1838
1839 IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1840
1841 setShadow(&I, getCleanShadow(&I));
1842 setOrigin(&I, getCleanOrigin());
1843 }
1844
1845 void visitAtomicRMWInst(AtomicRMWInst &I) {
1846 handleCASOrRMW(I);
1847 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1848 }
1849
1850 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1851 handleCASOrRMW(I);
1852 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1853 }
1854
1855 // Vector manipulation.
1856 void visitExtractElementInst(ExtractElementInst &I) {
1857 insertShadowCheck(I.getOperand(1), &I);
1858 IRBuilder<> IRB(&I);
1859 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1860 "_msprop"));
1861 setOrigin(&I, getOrigin(&I, 0));
1862 }
1863
1864 void visitInsertElementInst(InsertElementInst &I) {
1865 insertShadowCheck(I.getOperand(2), &I);
1866 IRBuilder<> IRB(&I);
1867 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1868 I.getOperand(2), "_msprop"));
1869 setOriginForNaryOp(I);
1870 }
1871
1872 void visitShuffleVectorInst(ShuffleVectorInst &I) {
1873 insertShadowCheck(I.getOperand(2), &I);
1874 IRBuilder<> IRB(&I);
1875 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1876 I.getOperand(2), "_msprop"));
1877 setOriginForNaryOp(I);
1878 }
1879
1880 // Casts.
1881 void visitSExtInst(SExtInst &I) {
1882 IRBuilder<> IRB(&I);
1883 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1884 setOrigin(&I, getOrigin(&I, 0));
1885 }
1886
1887 void visitZExtInst(ZExtInst &I) {
1888 IRBuilder<> IRB(&I);
1889 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1890 setOrigin(&I, getOrigin(&I, 0));
1891 }
1892
1893 void visitTruncInst(TruncInst &I) {
1894 IRBuilder<> IRB(&I);
1895 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1896 setOrigin(&I, getOrigin(&I, 0));
1897 }
1898
1899 void visitBitCastInst(BitCastInst &I) {
1900 // Special case: if this is the bitcast (there is exactly 1 allowed) between
1901 // a musttail call and a ret, don't instrument. New instructions are not
1902 // allowed after a musttail call.
1903 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1904 if (CI->isMustTailCall())
1905 return;
1906 IRBuilder<> IRB(&I);
1907 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1908 setOrigin(&I, getOrigin(&I, 0));
1909 }
1910
1911 void visitPtrToIntInst(PtrToIntInst &I) {
1912 IRBuilder<> IRB(&I);
1913 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1914 "_msprop_ptrtoint"));
1915 setOrigin(&I, getOrigin(&I, 0));
1916 }
1917
1918 void visitIntToPtrInst(IntToPtrInst &I) {
1919 IRBuilder<> IRB(&I);
1920 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1921 "_msprop_inttoptr"));
1922 setOrigin(&I, getOrigin(&I, 0));
1923 }
1924
1925 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
1926 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
1927 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
1928 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
1929 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
1930 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1931
1932 /// Propagate shadow for bitwise AND.
1933 ///
1934 /// This code is exact, i.e. if, for example, a bit in the left argument
1935 /// is defined and 0, then neither the value not definedness of the
1936 /// corresponding bit in B don't affect the resulting shadow.
1937 void visitAnd(BinaryOperator &I) {
1938 IRBuilder<> IRB(&I);
1939 // "And" of 0 and a poisoned value results in unpoisoned value.
1940 // 1&1 => 1; 0&1 => 0; p&1 => p;
1941 // 1&0 => 0; 0&0 => 0; p&0 => 0;
1942 // 1&p => p; 0&p => 0; p&p => p;
1943 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1944 Value *S1 = getShadow(&I, 0);
1945 Value *S2 = getShadow(&I, 1);
1946 Value *V1 = I.getOperand(0);
1947 Value *V2 = I.getOperand(1);
1948 if (V1->getType() != S1->getType()) {
1949 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1950 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1951 }
1952 Value *S1S2 = IRB.CreateAnd(S1, S2);
1953 Value *V1S2 = IRB.CreateAnd(V1, S2);
1954 Value *S1V2 = IRB.CreateAnd(S1, V2);
1955 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
1956 setOriginForNaryOp(I);
1957 }
1958
1959 void visitOr(BinaryOperator &I) {
1960 IRBuilder<> IRB(&I);
1961 // "Or" of 1 and a poisoned value results in unpoisoned value.
1962 // 1|1 => 1; 0|1 => 1; p|1 => 1;
1963 // 1|0 => 1; 0|0 => 0; p|0 => p;
1964 // 1|p => 1; 0|p => p; p|p => p;
1965 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1966 Value *S1 = getShadow(&I, 0);
1967 Value *S2 = getShadow(&I, 1);
1968 Value *V1 = IRB.CreateNot(I.getOperand(0));
1969 Value *V2 = IRB.CreateNot(I.getOperand(1));
1970 if (V1->getType() != S1->getType()) {
1971 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1972 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1973 }
1974 Value *S1S2 = IRB.CreateAnd(S1, S2);
1975 Value *V1S2 = IRB.CreateAnd(V1, S2);
1976 Value *S1V2 = IRB.CreateAnd(S1, V2);
1977 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
1978 setOriginForNaryOp(I);
1979 }
1980
1981 /// Default propagation of shadow and/or origin.
1982 ///
1983 /// This class implements the general case of shadow propagation, used in all
1984 /// cases where we don't know and/or don't care about what the operation
1985 /// actually does. It converts all input shadow values to a common type
1986 /// (extending or truncating as necessary), and bitwise OR's them.
1987 ///
1988 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1989 /// fully initialized), and less prone to false positives.
1990 ///
1991 /// This class also implements the general case of origin propagation. For a
1992 /// Nary operation, result origin is set to the origin of an argument that is
1993 /// not entirely initialized. If there is more than one such arguments, the
1994 /// rightmost of them is picked. It does not matter which one is picked if all
1995 /// arguments are initialized.
1996 template <bool CombineShadow>
1997 class Combiner {
1998 Value *Shadow = nullptr;
1999 Value *Origin = nullptr;
2000 IRBuilder<> &IRB;
2001 MemorySanitizerVisitor *MSV;
2002
2003 public:
2004 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2005 : IRB(IRB), MSV(MSV) {}
2006
2007 /// Add a pair of shadow and origin values to the mix.
2008 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2009 if (CombineShadow) {
2010 assert(OpShadow)((OpShadow) ? static_cast<void> (0) : __assert_fail ("OpShadow"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2010, __PRETTY_FUNCTION__))
;
2011 if (!Shadow)
2012 Shadow = OpShadow;
2013 else {
2014 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2015 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2016 }
2017 }
2018
2019 if (MSV->MS.TrackOrigins) {
2020 assert(OpOrigin)((OpOrigin) ? static_cast<void> (0) : __assert_fail ("OpOrigin"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2020, __PRETTY_FUNCTION__))
;
2021 if (!Origin) {
2022 Origin = OpOrigin;
2023 } else {
2024 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2025 // No point in adding something that might result in 0 origin value.
2026 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2027 Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
2028 Value *Cond =
2029 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2030 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2031 }
2032 }
2033 }
2034 return *this;
2035 }
2036
2037 /// Add an application value to the mix.
2038 Combiner &Add(Value *V) {
2039 Value *OpShadow = MSV->getShadow(V);
2040 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2041 return Add(OpShadow, OpOrigin);
2042 }
2043
2044 /// Set the current combined values as the given instruction's shadow
2045 /// and origin.
2046 void Done(Instruction *I) {
2047 if (CombineShadow) {
2048 assert(Shadow)((Shadow) ? static_cast<void> (0) : __assert_fail ("Shadow"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2048, __PRETTY_FUNCTION__))
;
2049 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2050 MSV->setShadow(I, Shadow);
2051 }
2052 if (MSV->MS.TrackOrigins) {
2053 assert(Origin)((Origin) ? static_cast<void> (0) : __assert_fail ("Origin"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2053, __PRETTY_FUNCTION__))
;
2054 MSV->setOrigin(I, Origin);
2055 }
2056 }
2057 };
2058
2059 using ShadowAndOriginCombiner = Combiner<true>;
2060 using OriginCombiner = Combiner<false>;
2061
2062 /// Propagate origin for arbitrary operation.
2063 void setOriginForNaryOp(Instruction &I) {
2064 if (!MS.TrackOrigins) return;
2065 IRBuilder<> IRB(&I);
2066 OriginCombiner OC(this, IRB);
2067 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
2068 OC.Add(OI->get());
2069 OC.Done(&I);
2070 }
2071
2072 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2073 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&((!(Ty->isVectorTy() && Ty->getScalarType()->
isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2074, __PRETTY_FUNCTION__))
2074 "Vector of pointers is not a valid shadow type")((!(Ty->isVectorTy() && Ty->getScalarType()->
isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? static_cast<void> (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2074, __PRETTY_FUNCTION__))
;
2075 return Ty->isVectorTy() ?
2076 Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
2077 Ty->getPrimitiveSizeInBits();
2078 }
2079
2080 /// Cast between two shadow types, extending or truncating as
2081 /// necessary.
2082 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2083 bool Signed = false) {
2084 Type *srcTy = V->getType();
2085 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2086 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2087 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2088 return IRB.CreateICmpNE(V, getCleanShadow(V));
2089
2090 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2091 return IRB.CreateIntCast(V, dstTy, Signed);
2092 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2093 dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
2094 return IRB.CreateIntCast(V, dstTy, Signed);
2095 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2096 Value *V2 =
2097 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2098 return IRB.CreateBitCast(V2, dstTy);
2099 // TODO: handle struct types.
2100 }
2101
2102 /// Cast an application value to the type of its own shadow.
2103 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2104 Type *ShadowTy = getShadowTy(V);
2105 if (V->getType() == ShadowTy)
2106 return V;
2107 if (V->getType()->isPtrOrPtrVectorTy())
2108 return IRB.CreatePtrToInt(V, ShadowTy);
2109 else
2110 return IRB.CreateBitCast(V, ShadowTy);
2111 }
2112
2113 /// Propagate shadow for arbitrary operation.
2114 void handleShadowOr(Instruction &I) {
2115 IRBuilder<> IRB(&I);
2116 ShadowAndOriginCombiner SC(this, IRB);
2117 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
2118 SC.Add(OI->get());
2119 SC.Done(&I);
2120 }
2121
2122 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2123
2124 // Handle multiplication by constant.
2125 //
2126 // Handle a special case of multiplication by constant that may have one or
2127 // more zeros in the lower bits. This makes corresponding number of lower bits
2128 // of the result zero as well. We model it by shifting the other operand
2129 // shadow left by the required number of bits. Effectively, we transform
2130 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2131 // We use multiplication by 2**N instead of shift to cover the case of
2132 // multiplication by 0, which may occur in some elements of a vector operand.
2133 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2134 Value *OtherArg) {
2135 Constant *ShadowMul;
2136 Type *Ty = ConstArg->getType();
2137 if (Ty->isVectorTy()) {
2138 unsigned NumElements = Ty->getVectorNumElements();
2139 Type *EltTy = Ty->getSequentialElementType();
2140 SmallVector<Constant *, 16> Elements;
2141 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2142 if (ConstantInt *Elt =
2143 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2144 const APInt &V = Elt->getValue();
2145 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2146 Elements.push_back(ConstantInt::get(EltTy, V2));
2147 } else {
2148 Elements.push_back(ConstantInt::get(EltTy, 1));
2149 }
2150 }
2151 ShadowMul = ConstantVector::get(Elements);
2152 } else {
2153 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2154 const APInt &V = Elt->getValue();
2155 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2156 ShadowMul = ConstantInt::get(Ty, V2);
2157 } else {
2158 ShadowMul = ConstantInt::get(Ty, 1);
2159 }
2160 }
2161
2162 IRBuilder<> IRB(&I);
2163 setShadow(&I,
2164 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2165 setOrigin(&I, getOrigin(OtherArg));
2166 }
2167
2168 void visitMul(BinaryOperator &I) {
2169 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2170 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2171 if (constOp0 && !constOp1)
2172 handleMulByConstant(I, constOp0, I.getOperand(1));
2173 else if (constOp1 && !constOp0)
2174 handleMulByConstant(I, constOp1, I.getOperand(0));
2175 else
2176 handleShadowOr(I);
2177 }
2178
2179 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2180 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2181 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2182 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2183 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2184 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2185
2186 void handleIntegerDiv(Instruction &I) {
2187 IRBuilder<> IRB(&I);
2188 // Strict on the second argument.
2189 insertShadowCheck(I.getOperand(1), &I);
2190 setShadow(&I, getShadow(&I, 0));
2191 setOrigin(&I, getOrigin(&I, 0));
2192 }
2193
2194 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2195 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2196 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2197 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2198
2199 // Floating point division is side-effect free. We can not require that the
2200 // divisor is fully initialized and must propagate shadow. See PR37523.
2201 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2202 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2203
2204 /// Instrument == and != comparisons.
2205 ///
2206 /// Sometimes the comparison result is known even if some of the bits of the
2207 /// arguments are not.
2208 void handleEqualityComparison(ICmpInst &I) {
2209 IRBuilder<> IRB(&I);
2210 Value *A = I.getOperand(0);
2211 Value *B = I.getOperand(1);
2212 Value *Sa = getShadow(A);
2213 Value *Sb = getShadow(B);
2214
2215 // Get rid of pointers and vectors of pointers.
2216 // For ints (and vectors of ints), types of A and Sa match,
2217 // and this is a no-op.
2218 A = IRB.CreatePointerCast(A, Sa->getType());
2219 B = IRB.CreatePointerCast(B, Sb->getType());
2220
2221 // A == B <==> (C = A^B) == 0
2222 // A != B <==> (C = A^B) != 0
2223 // Sc = Sa | Sb
2224 Value *C = IRB.CreateXor(A, B);
2225 Value *Sc = IRB.CreateOr(Sa, Sb);
2226 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2227 // Result is defined if one of the following is true
2228 // * there is a defined 1 bit in C
2229 // * C is fully defined
2230 // Si = !(C & ~Sc) && Sc
2231 Value *Zero = Constant::getNullValue(Sc->getType());
2232 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2233 Value *Si =
2234 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
2235 IRB.CreateICmpEQ(
2236 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2237 Si->setName("_msprop_icmp");
2238 setShadow(&I, Si);
2239 setOriginForNaryOp(I);
2240 }
2241
2242 /// Build the lowest possible value of V, taking into account V's
2243 /// uninitialized bits.
2244 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2245 bool isSigned) {
2246 if (isSigned) {
2247 // Split shadow into sign bit and other bits.
2248 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2249 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2250 // Maximise the undefined shadow bit, minimize other undefined bits.
2251 return
2252 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
2253 } else {
2254 // Minimize undefined bits.
2255 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2256 }
2257 }
2258
2259 /// Build the highest possible value of V, taking into account V's
2260 /// uninitialized bits.
2261 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2262 bool isSigned) {
2263 if (isSigned) {
2264 // Split shadow into sign bit and other bits.
2265 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2266 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2267 // Minimise the undefined shadow bit, maximise other undefined bits.
2268 return
2269 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
2270 } else {
2271 // Maximize undefined bits.
2272 return IRB.CreateOr(A, Sa);
2273 }
2274 }
2275
2276 /// Instrument relational comparisons.
2277 ///
2278 /// This function does exact shadow propagation for all relational
2279 /// comparisons of integers, pointers and vectors of those.
2280 /// FIXME: output seems suboptimal when one of the operands is a constant
2281 void handleRelationalComparisonExact(ICmpInst &I) {
2282 IRBuilder<> IRB(&I);
2283 Value *A = I.getOperand(0);
2284 Value *B = I.getOperand(1);
2285 Value *Sa = getShadow(A);
2286 Value *Sb = getShadow(B);
2287
2288 // Get rid of pointers and vectors of pointers.
2289 // For ints (and vectors of ints), types of A and Sa match,
2290 // and this is a no-op.
2291 A = IRB.CreatePointerCast(A, Sa->getType());
2292 B = IRB.CreatePointerCast(B, Sb->getType());
2293
2294 // Let [a0, a1] be the interval of possible values of A, taking into account
2295 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2296 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2297 bool IsSigned = I.isSigned();
2298 Value *S1 = IRB.CreateICmp(I.getPredicate(),
2299 getLowestPossibleValue(IRB, A, Sa, IsSigned),
2300 getHighestPossibleValue(IRB, B, Sb, IsSigned));
2301 Value *S2 = IRB.CreateICmp(I.getPredicate(),
2302 getHighestPossibleValue(IRB, A, Sa, IsSigned),
2303 getLowestPossibleValue(IRB, B, Sb, IsSigned));
2304 Value *Si = IRB.CreateXor(S1, S2);
2305 setShadow(&I, Si);
2306 setOriginForNaryOp(I);
2307 }
2308
2309 /// Instrument signed relational comparisons.
2310 ///
2311 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2312 /// bit of the shadow. Everything else is delegated to handleShadowOr().
2313 void handleSignedRelationalComparison(ICmpInst &I) {
2314 Constant *constOp;
2315 Value *op = nullptr;
2316 CmpInst::Predicate pre;
2317 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2318 op = I.getOperand(0);
2319 pre = I.getPredicate();
2320 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2321 op = I.getOperand(1);
2322 pre = I.getSwappedPredicate();
2323 } else {
2324 handleShadowOr(I);
2325 return;
2326 }
2327
2328 if ((constOp->isNullValue() &&
2329 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2330 (constOp->isAllOnesValue() &&
2331 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2332 IRBuilder<> IRB(&I);
2333 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2334 "_msprop_icmp_s");
2335 setShadow(&I, Shadow);
2336 setOrigin(&I, getOrigin(op));
2337 } else {
2338 handleShadowOr(I);
2339 }
2340 }
2341
2342 void visitICmpInst(ICmpInst &I) {
2343 if (!ClHandleICmp) {
2344 handleShadowOr(I);
2345 return;
2346 }
2347 if (I.isEquality()) {
2348 handleEqualityComparison(I);
2349 return;
2350 }
2351
2352 assert(I.isRelational())((I.isRelational()) ? static_cast<void> (0) : __assert_fail
("I.isRelational()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2352, __PRETTY_FUNCTION__))
;
2353 if (ClHandleICmpExact) {
2354 handleRelationalComparisonExact(I);
2355 return;
2356 }
2357 if (I.isSigned()) {
2358 handleSignedRelationalComparison(I);
2359 return;
2360 }
2361
2362 assert(I.isUnsigned())((I.isUnsigned()) ? static_cast<void> (0) : __assert_fail
("I.isUnsigned()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2362, __PRETTY_FUNCTION__))
;
2363 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2364 handleRelationalComparisonExact(I);
2365 return;
2366 }
2367
2368 handleShadowOr(I);
2369 }
2370
2371 void visitFCmpInst(FCmpInst &I) {
2372 handleShadowOr(I);
2373 }
2374
2375 void handleShift(BinaryOperator &I) {
2376 IRBuilder<> IRB(&I);
2377 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2378 // Otherwise perform the same shift on S1.
2379 Value *S1 = getShadow(&I, 0);
2380 Value *S2 = getShadow(&I, 1);
2381 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2382 S2->getType());
2383 Value *V2 = I.getOperand(1);
2384 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2385 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2386 setOriginForNaryOp(I);
2387 }
2388
2389 void visitShl(BinaryOperator &I) { handleShift(I); }
2390 void visitAShr(BinaryOperator &I) { handleShift(I); }
2391 void visitLShr(BinaryOperator &I) { handleShift(I); }
2392
2393 /// Instrument llvm.memmove
2394 ///
2395 /// At this point we don't know if llvm.memmove will be inlined or not.
2396 /// If we don't instrument it and it gets inlined,
2397 /// our interceptor will not kick in and we will lose the memmove.
2398 /// If we instrument the call here, but it does not get inlined,
2399 /// we will memove the shadow twice: which is bad in case
2400 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2401 ///
2402 /// Similar situation exists for memcpy and memset.
2403 void visitMemMoveInst(MemMoveInst &I) {
2404 IRBuilder<> IRB(&I);
2405 IRB.CreateCall(
2406 MS.MemmoveFn,
2407 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2408 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2409 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2410 I.eraseFromParent();
2411 }
2412
2413 // Similar to memmove: avoid copying shadow twice.
2414 // This is somewhat unfortunate as it may slowdown small constant memcpys.
2415 // FIXME: consider doing manual inline for small constant sizes and proper
2416 // alignment.
2417 void visitMemCpyInst(MemCpyInst &I) {
2418 IRBuilder<> IRB(&I);
2419 IRB.CreateCall(
2420 MS.MemcpyFn,
2421 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2422 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2423 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2424 I.eraseFromParent();
2425 }
2426
2427 // Same as memcpy.
2428 void visitMemSetInst(MemSetInst &I) {
2429 IRBuilder<> IRB(&I);
2430 IRB.CreateCall(
2431 MS.MemsetFn,
2432 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2433 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2434 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2435 I.eraseFromParent();
2436 }
2437
2438 void visitVAStartInst(VAStartInst &I) {
2439 VAHelper->visitVAStartInst(I);
2440 }
2441
2442 void visitVACopyInst(VACopyInst &I) {
2443 VAHelper->visitVACopyInst(I);
2444 }
2445
2446 /// Handle vector store-like intrinsics.
2447 ///
2448 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2449 /// has 1 pointer argument and 1 vector argument, returns void.
2450 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2451 IRBuilder<> IRB(&I);
2452 Value* Addr = I.getArgOperand(0);
2453 Value *Shadow = getShadow(&I, 1);
2454 Value *ShadowPtr, *OriginPtr;
2455
2456 // We don't know the pointer alignment (could be unaligned SSE store!).
2457 // Have to assume to worst case.
2458 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2459 Addr, IRB, Shadow->getType(), Align::None(), /*isStore*/ true);
2460 IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
2461
2462 if (ClCheckAccessAddress)
2463 insertShadowCheck(Addr, &I);
2464
2465 // FIXME: factor out common code from materializeStores
2466 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2467 return true;
2468 }
2469
2470 /// Handle vector load-like intrinsics.
2471 ///
2472 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2473 /// has 1 pointer argument, returns a vector.
2474 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2475 IRBuilder<> IRB(&I);
2476 Value *Addr = I.getArgOperand(0);
2477
2478 Type *ShadowTy = getShadowTy(&I);
2479 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2480 if (PropagateShadow) {
2481 // We don't know the pointer alignment (could be unaligned SSE load!).
2482 // Have to assume to worst case.
2483 const Align Alignment = Align::None();
2484 std::tie(ShadowPtr, OriginPtr) =
2485 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2486 setShadow(&I, IRB.CreateAlignedLoad(ShadowTy, ShadowPtr,
2487 Alignment.value(), "_msld"));
2488 } else {
2489 setShadow(&I, getCleanShadow(&I));
2490 }
2491
2492 if (ClCheckAccessAddress)
2493 insertShadowCheck(Addr, &I);
2494
2495 if (MS.TrackOrigins) {
2496 if (PropagateShadow)
2497 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2498 else
2499 setOrigin(&I, getCleanOrigin());
2500 }
2501 return true;
2502 }
2503
2504 /// Handle (SIMD arithmetic)-like intrinsics.
2505 ///
2506 /// Instrument intrinsics with any number of arguments of the same type,
2507 /// equal to the return type. The type should be simple (no aggregates or
2508 /// pointers; vectors are fine).
2509 /// Caller guarantees that this intrinsic does not access memory.
2510 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2511 Type *RetTy = I.getType();
2512 if (!(RetTy->isIntOrIntVectorTy() ||
2513 RetTy->isFPOrFPVectorTy() ||
2514 RetTy->isX86_MMXTy()))
2515 return false;
2516
2517 unsigned NumArgOperands = I.getNumArgOperands();
2518
2519 for (unsigned i = 0; i < NumArgOperands; ++i) {
2520 Type *Ty = I.getArgOperand(i)->getType();
2521 if (Ty != RetTy)
2522 return false;
2523 }
2524
2525 IRBuilder<> IRB(&I);
2526 ShadowAndOriginCombiner SC(this, IRB);
2527 for (unsigned i = 0; i < NumArgOperands; ++i)
2528 SC.Add(I.getArgOperand(i));
2529 SC.Done(&I);
2530
2531 return true;
2532 }
2533
2534 /// Heuristically instrument unknown intrinsics.
2535 ///
2536 /// The main purpose of this code is to do something reasonable with all
2537 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2538 /// We recognize several classes of intrinsics by their argument types and
2539 /// ModRefBehaviour and apply special intrumentation when we are reasonably
2540 /// sure that we know what the intrinsic does.
2541 ///
2542 /// We special-case intrinsics where this approach fails. See llvm.bswap
2543 /// handling as an example of that.
2544 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2545 unsigned NumArgOperands = I.getNumArgOperands();
2546 if (NumArgOperands == 0)
2547 return false;
2548
2549 if (NumArgOperands == 2 &&
2550 I.getArgOperand(0)->getType()->isPointerTy() &&
2551 I.getArgOperand(1)->getType()->isVectorTy() &&
2552 I.getType()->isVoidTy() &&
2553 !I.onlyReadsMemory()) {
2554 // This looks like a vector store.
2555 return handleVectorStoreIntrinsic(I);
2556 }
2557
2558 if (NumArgOperands == 1 &&
2559 I.getArgOperand(0)->getType()->isPointerTy() &&
2560 I.getType()->isVectorTy() &&
2561 I.onlyReadsMemory()) {
2562 // This looks like a vector load.
2563 return handleVectorLoadIntrinsic(I);
2564 }
2565
2566 if (I.doesNotAccessMemory())
2567 if (maybeHandleSimpleNomemIntrinsic(I))
2568 return true;
2569
2570 // FIXME: detect and handle SSE maskstore/maskload
2571 return false;
2572 }
2573
2574 void handleInvariantGroup(IntrinsicInst &I) {
2575 setShadow(&I, getShadow(&I, 0));
2576 setOrigin(&I, getOrigin(&I, 0));
2577 }
2578
2579 void handleLifetimeStart(IntrinsicInst &I) {
2580 if (!PoisonStack)
2581 return;
2582 DenseMap<Value *, AllocaInst *> AllocaForValue;
2583 AllocaInst *AI =
2584 llvm::findAllocaForValue(I.getArgOperand(1), AllocaForValue);
2585 if (!AI)
2586 InstrumentLifetimeStart = false;
2587 LifetimeStartList.push_back(std::make_pair(&I, AI));
2588 }
2589
2590 void handleBswap(IntrinsicInst &I) {
2591 IRBuilder<> IRB(&I);
2592 Value *Op = I.getArgOperand(0);
2593 Type *OpType = Op->getType();
2594 Function *BswapFunc = Intrinsic::getDeclaration(
2595 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2596 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2597 setOrigin(&I, getOrigin(Op));
2598 }
2599
2600 // Instrument vector convert instrinsic.
2601 //
2602 // This function instruments intrinsics like cvtsi2ss:
2603 // %Out = int_xxx_cvtyyy(%ConvertOp)
2604 // or
2605 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2606 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2607 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2608 // elements from \p CopyOp.
2609 // In most cases conversion involves floating-point value which may trigger a
2610 // hardware exception when not fully initialized. For this reason we require
2611 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2612 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2613 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2614 // return a fully initialized value.
2615 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2616 IRBuilder<> IRB(&I);
2617 Value *CopyOp, *ConvertOp;
2618
2619 switch (I.getNumArgOperands()) {
2620 case 3:
2621 assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode")((isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantInt>(I.getArgOperand(2)) && \"Invalid rounding mode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2621, __PRETTY_FUNCTION__))
;
2622 LLVM_FALLTHROUGH[[gnu::fallthrough]];
2623 case 2:
2624 CopyOp = I.getArgOperand(0);
2625 ConvertOp = I.getArgOperand(1);
2626 break;
2627 case 1:
2628 ConvertOp = I.getArgOperand(0);
2629 CopyOp = nullptr;
2630 break;
2631 default:
2632 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.")::llvm::llvm_unreachable_internal("Cvt intrinsic with unsupported number of arguments."
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2632)
;
2633 }
2634
2635 // The first *NumUsedElements* elements of ConvertOp are converted to the
2636 // same number of output elements. The rest of the output is copied from
2637 // CopyOp, or (if not available) filled with zeroes.
2638 // Combine shadow for elements of ConvertOp that are used in this operation,
2639 // and insert a check.
2640 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2641 // int->any conversion.
2642 Value *ConvertShadow = getShadow(ConvertOp);
2643 Value *AggShadow = nullptr;
2644 if (ConvertOp->getType()->isVectorTy()) {
2645 AggShadow = IRB.CreateExtractElement(
2646 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2647 for (int i = 1; i < NumUsedElements; ++i) {
2648 Value *MoreShadow = IRB.CreateExtractElement(
2649 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2650 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2651 }
2652 } else {
2653 AggShadow = ConvertShadow;
2654 }
2655 assert(AggShadow->getType()->isIntegerTy())((AggShadow->getType()->isIntegerTy()) ? static_cast<
void> (0) : __assert_fail ("AggShadow->getType()->isIntegerTy()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2655, __PRETTY_FUNCTION__))
;
2656 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2657
2658 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2659 // ConvertOp.
2660 if (CopyOp) {
2661 assert(CopyOp->getType() == I.getType())((CopyOp->getType() == I.getType()) ? static_cast<void>
(0) : __assert_fail ("CopyOp->getType() == I.getType()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2661, __PRETTY_FUNCTION__))
;
2662 assert(CopyOp->getType()->isVectorTy())((CopyOp->getType()->isVectorTy()) ? static_cast<void
> (0) : __assert_fail ("CopyOp->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2662, __PRETTY_FUNCTION__))
;
2663 Value *ResultShadow = getShadow(CopyOp);
2664 Type *EltTy = ResultShadow->getType()->getVectorElementType();
2665 for (int i = 0; i < NumUsedElements; ++i) {
2666 ResultShadow = IRB.CreateInsertElement(
2667 ResultShadow, ConstantInt::getNullValue(EltTy),
2668 ConstantInt::get(IRB.getInt32Ty(), i));
2669 }
2670 setShadow(&I, ResultShadow);
2671 setOrigin(&I, getOrigin(CopyOp));
2672 } else {
2673 setShadow(&I, getCleanShadow(&I));
2674 setOrigin(&I, getCleanOrigin());
2675 }
2676 }
2677
2678 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2679 // zeroes if it is zero, and all ones otherwise.
2680 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2681 if (S->getType()->isVectorTy())
2682 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2683 assert(S->getType()->getPrimitiveSizeInBits() <= 64)((S->getType()->getPrimitiveSizeInBits() <= 64) ? static_cast
<void> (0) : __assert_fail ("S->getType()->getPrimitiveSizeInBits() <= 64"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2683, __PRETTY_FUNCTION__))
;
2684 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2685 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2686 }
2687
2688 // Given a vector, extract its first element, and return all
2689 // zeroes if it is zero, and all ones otherwise.
2690 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2691 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2692 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2693 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2694 }
2695
2696 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2697 Type *T = S->getType();
2698 assert(T->isVectorTy())((T->isVectorTy()) ? static_cast<void> (0) : __assert_fail
("T->isVectorTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2698, __PRETTY_FUNCTION__))
;
2699 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2700 return IRB.CreateSExt(S2, T);
2701 }
2702
2703 // Instrument vector shift instrinsic.
2704 //
2705 // This function instruments intrinsics like int_x86_avx2_psll_w.
2706 // Intrinsic shifts %In by %ShiftSize bits.
2707 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2708 // size, and the rest is ignored. Behavior is defined even if shift size is
2709 // greater than register (or field) width.
2710 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2711 assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) :
__assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2711, __PRETTY_FUNCTION__))
;
2712 IRBuilder<> IRB(&I);
2713 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2714 // Otherwise perform the same shift on S1.
2715 Value *S1 = getShadow(&I, 0);
2716 Value *S2 = getShadow(&I, 1);
2717 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2718 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2719 Value *V1 = I.getOperand(0);
2720 Value *V2 = I.getOperand(1);
2721 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledValue(),
2722 {IRB.CreateBitCast(S1, V1->getType()), V2});
2723 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2724 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2725 setOriginForNaryOp(I);
2726 }
2727
2728 // Get an X86_MMX-sized vector type.
2729 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2730 const unsigned X86_MMXSizeInBits = 64;
2731 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits
) == 0 && "Illegal MMX vector element size") ? static_cast
<void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2732, __PRETTY_FUNCTION__))
2732 "Illegal MMX vector element size")((EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits
) == 0 && "Illegal MMX vector element size") ? static_cast
<void> (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2732, __PRETTY_FUNCTION__))
;
2733 return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2734 X86_MMXSizeInBits / EltSizeInBits);
2735 }
2736
2737 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2738 // intrinsic.
2739 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2740 switch (id) {
2741 case Intrinsic::x86_sse2_packsswb_128:
2742 case Intrinsic::x86_sse2_packuswb_128:
2743 return Intrinsic::x86_sse2_packsswb_128;
2744
2745 case Intrinsic::x86_sse2_packssdw_128:
2746 case Intrinsic::x86_sse41_packusdw:
2747 return Intrinsic::x86_sse2_packssdw_128;
2748
2749 case Intrinsic::x86_avx2_packsswb:
2750 case Intrinsic::x86_avx2_packuswb:
2751 return Intrinsic::x86_avx2_packsswb;
2752
2753 case Intrinsic::x86_avx2_packssdw:
2754 case Intrinsic::x86_avx2_packusdw:
2755 return Intrinsic::x86_avx2_packssdw;
2756
2757 case Intrinsic::x86_mmx_packsswb:
2758 case Intrinsic::x86_mmx_packuswb:
2759 return Intrinsic::x86_mmx_packsswb;
2760
2761 case Intrinsic::x86_mmx_packssdw:
2762 return Intrinsic::x86_mmx_packssdw;
2763 default:
2764 llvm_unreachable("unexpected intrinsic id")::llvm::llvm_unreachable_internal("unexpected intrinsic id", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2764)
;
2765 }
2766 }
2767
2768 // Instrument vector pack instrinsic.
2769 //
2770 // This function instruments intrinsics like x86_mmx_packsswb, that
2771 // packs elements of 2 input vectors into half as many bits with saturation.
2772 // Shadow is propagated with the signed variant of the same intrinsic applied
2773 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2774 // EltSizeInBits is used only for x86mmx arguments.
2775 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2776 assert(I.getNumArgOperands() == 2)((I.getNumArgOperands() == 2) ? static_cast<void> (0) :
__assert_fail ("I.getNumArgOperands() == 2", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2776, __PRETTY_FUNCTION__))
;
2777 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2778 IRBuilder<> IRB(&I);
2779 Value *S1 = getShadow(&I, 0);
2780 Value *S2 = getShadow(&I, 1);
2781 assert(isX86_MMX || S1->getType()->isVectorTy())((isX86_MMX || S1->getType()->isVectorTy()) ? static_cast
<void> (0) : __assert_fail ("isX86_MMX || S1->getType()->isVectorTy()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2781, __PRETTY_FUNCTION__))
;
2782
2783 // SExt and ICmpNE below must apply to individual elements of input vectors.
2784 // In case of x86mmx arguments, cast them to appropriate vector types and
2785 // back.
2786 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2787 if (isX86_MMX) {
2788 S1 = IRB.CreateBitCast(S1, T);
2789 S2 = IRB.CreateBitCast(S2, T);
2790 }
2791 Value *S1_ext = IRB.CreateSExt(
2792 IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2793 Value *S2_ext = IRB.CreateSExt(
2794 IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2795 if (isX86_MMX) {
2796 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2797 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2798 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2799 }
2800
2801 Function *ShadowFn = Intrinsic::getDeclaration(
2802 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2803
2804 Value *S =
2805 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2806 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2807 setShadow(&I, S);
2808 setOriginForNaryOp(I);
2809 }
2810
2811 // Instrument sum-of-absolute-differencies intrinsic.
2812 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2813 const unsigned SignificantBitsPerResultElement = 16;
2814 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2815 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2816 unsigned ZeroBitsPerResultElement =
2817 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2818
2819 IRBuilder<> IRB(&I);
2820 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2821 S = IRB.CreateBitCast(S, ResTy);
2822 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2823 ResTy);
2824 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2825 S = IRB.CreateBitCast(S, getShadowTy(&I));
2826 setShadow(&I, S);
2827 setOriginForNaryOp(I);
2828 }
2829
2830 // Instrument multiply-add intrinsic.
2831 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2832 unsigned EltSizeInBits = 0) {
2833 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2834 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2835 IRBuilder<> IRB(&I);
2836 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2837 S = IRB.CreateBitCast(S, ResTy);
2838 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2839 ResTy);
2840 S = IRB.CreateBitCast(S, getShadowTy(&I));
2841 setShadow(&I, S);
2842 setOriginForNaryOp(I);
2843 }
2844
2845 // Instrument compare-packed intrinsic.
2846 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2847 // all-ones shadow.
2848 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2849 IRBuilder<> IRB(&I);
2850 Type *ResTy = getShadowTy(&I);
2851 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2852 Value *S = IRB.CreateSExt(
2853 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2854 setShadow(&I, S);
2855 setOriginForNaryOp(I);
2856 }
2857
2858 // Instrument compare-scalar intrinsic.
2859 // This handles both cmp* intrinsics which return the result in the first
2860 // element of a vector, and comi* which return the result as i32.
2861 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2862 IRBuilder<> IRB(&I);
2863 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2864 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2865 setShadow(&I, S);
2866 setOriginForNaryOp(I);
2867 }
2868
2869 void handleStmxcsr(IntrinsicInst &I) {
2870 IRBuilder<> IRB(&I);
2871 Value* Addr = I.getArgOperand(0);
2872 Type *Ty = IRB.getInt32Ty();
2873 Value *ShadowPtr =
2874 getShadowOriginPtr(Addr, IRB, Ty, Align::None(), /*isStore*/ true)
2875 .first;
2876
2877 IRB.CreateStore(getCleanShadow(Ty),
2878 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
2879
2880 if (ClCheckAccessAddress)
2881 insertShadowCheck(Addr, &I);
2882 }
2883
2884 void handleLdmxcsr(IntrinsicInst &I) {
2885 if (!InsertChecks) return;
2886
2887 IRBuilder<> IRB(&I);
2888 Value *Addr = I.getArgOperand(0);
2889 Type *Ty = IRB.getInt32Ty();
2890 const Align Alignment = Align::None();
2891 Value *ShadowPtr, *OriginPtr;
2892 std::tie(ShadowPtr, OriginPtr) =
2893 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
2894
2895 if (ClCheckAccessAddress)
2896 insertShadowCheck(Addr, &I);
2897
2898 Value *Shadow =
2899 IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment.value(), "_ldmxcsr");
2900 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
2901 : getCleanOrigin();
2902 insertShadowCheck(Shadow, Origin, &I);
2903 }
2904
2905 void handleMaskedStore(IntrinsicInst &I) {
2906 IRBuilder<> IRB(&I);
2907 Value *V = I.getArgOperand(0);
2908 Value *Addr = I.getArgOperand(1);
2909 const MaybeAlign Alignment(
2910 cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
2911 Value *Mask = I.getArgOperand(3);
2912 Value *Shadow = getShadow(V);
2913
2914 Value *ShadowPtr;
2915 Value *OriginPtr;
2916 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2917 Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
2918
2919 if (ClCheckAccessAddress) {
2920 insertShadowCheck(Addr, &I);
2921 // Uninitialized mask is kind of like uninitialized address, but not as
2922 // scary.
2923 insertShadowCheck(Mask, &I);
2924 }
2925
2926 IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment ? Alignment->value() : 0,
2927 Mask);
2928
2929 if (MS.TrackOrigins) {
2930 auto &DL = F.getParent()->getDataLayout();
2931 paintOrigin(IRB, getOrigin(V), OriginPtr,
2932 DL.getTypeStoreSize(Shadow->getType()),
2933 llvm::max(Alignment, kMinOriginAlignment));
2934 }
2935 }
2936
2937 bool handleMaskedLoad(IntrinsicInst &I) {
2938 IRBuilder<> IRB(&I);
2939 Value *Addr = I.getArgOperand(0);
2940 const MaybeAlign Alignment(
2941 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
2942 Value *Mask = I.getArgOperand(2);
2943 Value *PassThru = I.getArgOperand(3);
2944
2945 Type *ShadowTy = getShadowTy(&I);
2946 Value *ShadowPtr, *OriginPtr;
2947 if (PropagateShadow) {
2948 std::tie(ShadowPtr, OriginPtr) =
2949 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2950 setShadow(&I, IRB.CreateMaskedLoad(
2951 ShadowPtr, Alignment ? Alignment->value() : 0, Mask,
2952 getShadow(PassThru), "_msmaskedld"));
2953 } else {
2954 setShadow(&I, getCleanShadow(&I));
2955 }
2956
2957 if (ClCheckAccessAddress) {
2958 insertShadowCheck(Addr, &I);
2959 insertShadowCheck(Mask, &I);
2960 }
2961
2962 if (MS.TrackOrigins) {
2963 if (PropagateShadow) {
2964 // Choose between PassThru's and the loaded value's origins.
2965 Value *MaskedPassThruShadow = IRB.CreateAnd(
2966 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
2967
2968 Value *Acc = IRB.CreateExtractElement(
2969 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2970 for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N;
2971 ++i) {
2972 Value *More = IRB.CreateExtractElement(
2973 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2974 Acc = IRB.CreateOr(Acc, More);
2975 }
2976
2977 Value *Origin = IRB.CreateSelect(
2978 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
2979 getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
2980
2981 setOrigin(&I, Origin);
2982 } else {
2983 setOrigin(&I, getCleanOrigin());
2984 }
2985 }
2986 return true;
2987 }
2988
2989 // Instrument BMI / BMI2 intrinsics.
2990 // All of these intrinsics are Z = I(X, Y)
2991 // where the types of all operands and the result match, and are either i32 or i64.
2992 // The following instrumentation happens to work for all of them:
2993 // Sz = I(Sx, Y) | (sext (Sy != 0))
2994 void handleBmiIntrinsic(IntrinsicInst &I) {
2995 IRBuilder<> IRB(&I);
2996 Type *ShadowTy = getShadowTy(&I);
2997
2998 // If any bit of the mask operand is poisoned, then the whole thing is.
2999 Value *SMask = getShadow(&I, 1);
3000 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3001 ShadowTy);
3002 // Apply the same intrinsic to the shadow of the first operand.
3003 Value *S = IRB.CreateCall(I.getCalledFunction(),
3004 {getShadow(&I, 0), I.getOperand(1)});
3005 S = IRB.CreateOr(SMask, S);
3006 setShadow(&I, S);
3007 setOriginForNaryOp(I);
3008 }
3009
3010 void visitIntrinsicInst(IntrinsicInst &I) {
3011 switch (I.getIntrinsicID()) {
3012 case Intrinsic::lifetime_start:
3013 handleLifetimeStart(I);
3014 break;
3015 case Intrinsic::launder_invariant_group:
3016 case Intrinsic::strip_invariant_group:
3017 handleInvariantGroup(I);
3018 break;
3019 case Intrinsic::bswap:
3020 handleBswap(I);
3021 break;
3022 case Intrinsic::masked_store:
3023 handleMaskedStore(I);
3024 break;
3025 case Intrinsic::masked_load:
3026 handleMaskedLoad(I);
3027 break;
3028 case Intrinsic::x86_sse_stmxcsr:
3029 handleStmxcsr(I);
3030 break;
3031 case Intrinsic::x86_sse_ldmxcsr:
3032 handleLdmxcsr(I);
3033 break;
3034 case Intrinsic::x86_avx512_vcvtsd2usi64:
3035 case Intrinsic::x86_avx512_vcvtsd2usi32:
3036 case Intrinsic::x86_avx512_vcvtss2usi64:
3037 case Intrinsic::x86_avx512_vcvtss2usi32:
3038 case Intrinsic::x86_avx512_cvttss2usi64:
3039 case Intrinsic::x86_avx512_cvttss2usi:
3040 case Intrinsic::x86_avx512_cvttsd2usi64:
3041 case Intrinsic::x86_avx512_cvttsd2usi:
3042 case Intrinsic::x86_avx512_cvtusi2ss:
3043 case Intrinsic::x86_avx512_cvtusi642sd:
3044 case Intrinsic::x86_avx512_cvtusi642ss:
3045 case Intrinsic::x86_sse2_cvtsd2si64:
3046 case Intrinsic::x86_sse2_cvtsd2si:
3047 case Intrinsic::x86_sse2_cvtsd2ss:
3048 case Intrinsic::x86_sse2_cvttsd2si64:
3049 case Intrinsic::x86_sse2_cvttsd2si:
3050 case Intrinsic::x86_sse_cvtss2si64:
3051 case Intrinsic::x86_sse_cvtss2si:
3052 case Intrinsic::x86_sse_cvttss2si64:
3053 case Intrinsic::x86_sse_cvttss2si:
3054 handleVectorConvertIntrinsic(I, 1);
3055 break;
3056 case Intrinsic::x86_sse_cvtps2pi:
3057 case Intrinsic::x86_sse_cvttps2pi:
3058 handleVectorConvertIntrinsic(I, 2);
3059 break;
3060
3061 case Intrinsic::x86_avx512_psll_w_512:
3062 case Intrinsic::x86_avx512_psll_d_512:
3063 case Intrinsic::x86_avx512_psll_q_512:
3064 case Intrinsic::x86_avx512_pslli_w_512:
3065 case Intrinsic::x86_avx512_pslli_d_512:
3066 case Intrinsic::x86_avx512_pslli_q_512:
3067 case Intrinsic::x86_avx512_psrl_w_512:
3068 case Intrinsic::x86_avx512_psrl_d_512:
3069 case Intrinsic::x86_avx512_psrl_q_512:
3070 case Intrinsic::x86_avx512_psra_w_512:
3071 case Intrinsic::x86_avx512_psra_d_512:
3072 case Intrinsic::x86_avx512_psra_q_512:
3073 case Intrinsic::x86_avx512_psrli_w_512:
3074 case Intrinsic::x86_avx512_psrli_d_512:
3075 case Intrinsic::x86_avx512_psrli_q_512:
3076 case Intrinsic::x86_avx512_psrai_w_512:
3077 case Intrinsic::x86_avx512_psrai_d_512:
3078 case Intrinsic::x86_avx512_psrai_q_512:
3079 case Intrinsic::x86_avx512_psra_q_256:
3080 case Intrinsic::x86_avx512_psra_q_128:
3081 case Intrinsic::x86_avx512_psrai_q_256:
3082 case Intrinsic::x86_avx512_psrai_q_128:
3083 case Intrinsic::x86_avx2_psll_w:
3084 case Intrinsic::x86_avx2_psll_d:
3085 case Intrinsic::x86_avx2_psll_q:
3086 case Intrinsic::x86_avx2_pslli_w:
3087 case Intrinsic::x86_avx2_pslli_d:
3088 case Intrinsic::x86_avx2_pslli_q:
3089 case Intrinsic::x86_avx2_psrl_w:
3090 case Intrinsic::x86_avx2_psrl_d:
3091 case Intrinsic::x86_avx2_psrl_q:
3092 case Intrinsic::x86_avx2_psra_w:
3093 case Intrinsic::x86_avx2_psra_d:
3094 case Intrinsic::x86_avx2_psrli_w:
3095 case Intrinsic::x86_avx2_psrli_d:
3096 case Intrinsic::x86_avx2_psrli_q:
3097 case Intrinsic::x86_avx2_psrai_w:
3098 case Intrinsic::x86_avx2_psrai_d:
3099 case Intrinsic::x86_sse2_psll_w:
3100 case Intrinsic::x86_sse2_psll_d:
3101 case Intrinsic::x86_sse2_psll_q:
3102 case Intrinsic::x86_sse2_pslli_w:
3103 case Intrinsic::x86_sse2_pslli_d:
3104 case Intrinsic::x86_sse2_pslli_q:
3105 case Intrinsic::x86_sse2_psrl_w:
3106 case Intrinsic::x86_sse2_psrl_d:
3107 case Intrinsic::x86_sse2_psrl_q:
3108 case Intrinsic::x86_sse2_psra_w:
3109 case Intrinsic::x86_sse2_psra_d:
3110 case Intrinsic::x86_sse2_psrli_w:
3111 case Intrinsic::x86_sse2_psrli_d:
3112 case Intrinsic::x86_sse2_psrli_q:
3113 case Intrinsic::x86_sse2_psrai_w:
3114 case Intrinsic::x86_sse2_psrai_d:
3115 case Intrinsic::x86_mmx_psll_w:
3116 case Intrinsic::x86_mmx_psll_d:
3117 case Intrinsic::x86_mmx_psll_q:
3118 case Intrinsic::x86_mmx_pslli_w:
3119 case Intrinsic::x86_mmx_pslli_d:
3120 case Intrinsic::x86_mmx_pslli_q:
3121 case Intrinsic::x86_mmx_psrl_w:
3122 case Intrinsic::x86_mmx_psrl_d:
3123 case Intrinsic::x86_mmx_psrl_q:
3124 case Intrinsic::x86_mmx_psra_w:
3125 case Intrinsic::x86_mmx_psra_d:
3126 case Intrinsic::x86_mmx_psrli_w:
3127 case Intrinsic::x86_mmx_psrli_d:
3128 case Intrinsic::x86_mmx_psrli_q:
3129 case Intrinsic::x86_mmx_psrai_w:
3130 case Intrinsic::x86_mmx_psrai_d:
3131 handleVectorShiftIntrinsic(I, /* Variable */ false);
3132 break;
3133 case Intrinsic::x86_avx2_psllv_d:
3134 case Intrinsic::x86_avx2_psllv_d_256:
3135 case Intrinsic::x86_avx512_psllv_d_512:
3136 case Intrinsic::x86_avx2_psllv_q:
3137 case Intrinsic::x86_avx2_psllv_q_256:
3138 case Intrinsic::x86_avx512_psllv_q_512:
3139 case Intrinsic::x86_avx2_psrlv_d:
3140 case Intrinsic::x86_avx2_psrlv_d_256:
3141 case Intrinsic::x86_avx512_psrlv_d_512:
3142 case Intrinsic::x86_avx2_psrlv_q:
3143 case Intrinsic::x86_avx2_psrlv_q_256:
3144 case Intrinsic::x86_avx512_psrlv_q_512:
3145 case Intrinsic::x86_avx2_psrav_d:
3146 case Intrinsic::x86_avx2_psrav_d_256:
3147 case Intrinsic::x86_avx512_psrav_d_512:
3148 case Intrinsic::x86_avx512_psrav_q_128:
3149 case Intrinsic::x86_avx512_psrav_q_256:
3150 case Intrinsic::x86_avx512_psrav_q_512:
3151 handleVectorShiftIntrinsic(I, /* Variable */ true);
3152 break;
3153
3154 case Intrinsic::x86_sse2_packsswb_128:
3155 case Intrinsic::x86_sse2_packssdw_128:
3156 case Intrinsic::x86_sse2_packuswb_128:
3157 case Intrinsic::x86_sse41_packusdw:
3158 case Intrinsic::x86_avx2_packsswb:
3159 case Intrinsic::x86_avx2_packssdw:
3160 case Intrinsic::x86_avx2_packuswb:
3161 case Intrinsic::x86_avx2_packusdw:
3162 handleVectorPackIntrinsic(I);
3163 break;
3164
3165 case Intrinsic::x86_mmx_packsswb:
3166 case Intrinsic::x86_mmx_packuswb:
3167 handleVectorPackIntrinsic(I, 16);
3168 break;
3169
3170 case Intrinsic::x86_mmx_packssdw:
3171 handleVectorPackIntrinsic(I, 32);
3172 break;
3173
3174 case Intrinsic::x86_mmx_psad_bw:
3175 case Intrinsic::x86_sse2_psad_bw:
3176 case Intrinsic::x86_avx2_psad_bw:
3177 handleVectorSadIntrinsic(I);
3178 break;
3179
3180 case Intrinsic::x86_sse2_pmadd_wd:
3181 case Intrinsic::x86_avx2_pmadd_wd:
3182 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3183 case Intrinsic::x86_avx2_pmadd_ub_sw:
3184 handleVectorPmaddIntrinsic(I);
3185 break;
3186
3187 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3188 handleVectorPmaddIntrinsic(I, 8);
3189 break;
3190
3191 case Intrinsic::x86_mmx_pmadd_wd:
3192 handleVectorPmaddIntrinsic(I, 16);
3193 break;
3194
3195 case Intrinsic::x86_sse_cmp_ss:
3196 case Intrinsic::x86_sse2_cmp_sd:
3197 case Intrinsic::x86_sse_comieq_ss:
3198 case Intrinsic::x86_sse_comilt_ss:
3199 case Intrinsic::x86_sse_comile_ss:
3200 case Intrinsic::x86_sse_comigt_ss:
3201 case Intrinsic::x86_sse_comige_ss:
3202 case Intrinsic::x86_sse_comineq_ss:
3203 case Intrinsic::x86_sse_ucomieq_ss:
3204 case Intrinsic::x86_sse_ucomilt_ss:
3205 case Intrinsic::x86_sse_ucomile_ss:
3206 case Intrinsic::x86_sse_ucomigt_ss:
3207 case Intrinsic::x86_sse_ucomige_ss:
3208 case Intrinsic::x86_sse_ucomineq_ss:
3209 case Intrinsic::x86_sse2_comieq_sd:
3210 case Intrinsic::x86_sse2_comilt_sd:
3211 case Intrinsic::x86_sse2_comile_sd:
3212 case Intrinsic::x86_sse2_comigt_sd:
3213 case Intrinsic::x86_sse2_comige_sd:
3214 case Intrinsic::x86_sse2_comineq_sd:
3215 case Intrinsic::x86_sse2_ucomieq_sd:
3216 case Intrinsic::x86_sse2_ucomilt_sd:
3217 case Intrinsic::x86_sse2_ucomile_sd:
3218 case Intrinsic::x86_sse2_ucomigt_sd:
3219 case Intrinsic::x86_sse2_ucomige_sd:
3220 case Intrinsic::x86_sse2_ucomineq_sd:
3221 handleVectorCompareScalarIntrinsic(I);
3222 break;
3223
3224 case Intrinsic::x86_sse_cmp_ps:
3225 case Intrinsic::x86_sse2_cmp_pd:
3226 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3227 // generates reasonably looking IR that fails in the backend with "Do not
3228 // know how to split the result of this operator!".
3229 handleVectorComparePackedIntrinsic(I);
3230 break;
3231
3232 case Intrinsic::x86_bmi_bextr_32:
3233 case Intrinsic::x86_bmi_bextr_64:
3234 case Intrinsic::x86_bmi_bzhi_32:
3235 case Intrinsic::x86_bmi_bzhi_64:
3236 case Intrinsic::x86_bmi_pdep_32:
3237 case Intrinsic::x86_bmi_pdep_64:
3238 case Intrinsic::x86_bmi_pext_32:
3239 case Intrinsic::x86_bmi_pext_64:
3240 handleBmiIntrinsic(I);
3241 break;
3242
3243 case Intrinsic::is_constant:
3244 // The result of llvm.is.constant() is always defined.
3245 setShadow(&I, getCleanShadow(&I));
3246 setOrigin(&I, getCleanOrigin());
3247 break;
3248
3249 default:
3250 if (!handleUnknownIntrinsic(I))
3251 visitInstruction(I);
3252 break;
3253 }
3254 }
3255
3256 void visitCallSite(CallSite CS) {
3257 Instruction &I = *CS.getInstruction();
3258 assert(!I.getMetadata("nosanitize"))((!I.getMetadata("nosanitize")) ? static_cast<void> (0)
: __assert_fail ("!I.getMetadata(\"nosanitize\")", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3258, __PRETTY_FUNCTION__))
;
3259 assert((CS.isCall() || CS.isInvoke() || CS.isCallBr()) &&(((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && "Unknown type of CallSite"
) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke() || CS.isCallBr()) && \"Unknown type of CallSite\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3260, __PRETTY_FUNCTION__))
3260 "Unknown type of CallSite")(((CS.isCall() || CS.isInvoke() || CS.isCallBr()) && "Unknown type of CallSite"
) ? static_cast<void> (0) : __assert_fail ("(CS.isCall() || CS.isInvoke() || CS.isCallBr()) && \"Unknown type of CallSite\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3260, __PRETTY_FUNCTION__))
;
3261 if (CS.isCallBr() || (CS.isCall() && cast<CallInst>(&I)->isInlineAsm())) {
3262 // For inline asm (either a call to asm function, or callbr instruction),
3263 // do the usual thing: check argument shadow and mark all outputs as
3264 // clean. Note that any side effects of the inline asm that are not
3265 // immediately visible in its constraints are not handled.
3266 if (ClHandleAsmConservative && MS.CompileKernel)
3267 visitAsmInstruction(I);
3268 else
3269 visitInstruction(I);
3270 return;
3271 }
3272 if (CS.isCall()) {
3273 CallInst *Call = cast<CallInst>(&I);
3274 assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere")((!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("!isa<IntrinsicInst>(&I) && \"intrinsics are handled elsewhere\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3274, __PRETTY_FUNCTION__))
;
3275
3276 // We are going to insert code that relies on the fact that the callee
3277 // will become a non-readonly function after it is instrumented by us. To
3278 // prevent this code from being optimized out, mark that function
3279 // non-readonly in advance.
3280 if (Function *Func = Call->getCalledFunction()) {
3281 // Clear out readonly/readnone attributes.
3282 AttrBuilder B;
3283 B.addAttribute(Attribute::ReadOnly)
3284 .addAttribute(Attribute::ReadNone)
3285 .addAttribute(Attribute::WriteOnly)
3286 .addAttribute(Attribute::ArgMemOnly)
3287 .addAttribute(Attribute::Speculatable);
3288 Func->removeAttributes(AttributeList::FunctionIndex, B);
3289 }
3290
3291 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
3292 }
3293 IRBuilder<> IRB(&I);
3294
3295 unsigned ArgOffset = 0;
3296 LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CallSite: " << I <<
"\n"; } } while (false)
;
3297 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3298 ArgIt != End; ++ArgIt) {
3299 Value *A = *ArgIt;
3300 unsigned i = ArgIt - CS.arg_begin();
3301 if (!A->getType()->isSized()) {
3302 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg " << i << " is not sized: "
<< I << "\n"; } } while (false)
;
3303 continue;
3304 }
3305 unsigned Size = 0;
3306 Value *Store = nullptr;
3307 // Compute the Shadow for arg even if it is ByVal, because
3308 // in that case getShadow() will copy the actual arg shadow to
3309 // __msan_param_tls.
3310 Value *ArgShadow = getShadow(A);
3311 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3312 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *Ado { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
3313 << " Shadow: " << *ArgShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
;
3314 bool ArgIsInitialized = false;
3315 const DataLayout &DL = F.getParent()->getDataLayout();
3316 if (CS.paramHasAttr(i, Attribute::ByVal)) {
3317 assert(A->getType()->isPointerTy() &&((A->getType()->isPointerTy() && "ByVal argument is not a pointer!"
) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3318, __PRETTY_FUNCTION__))
3318 "ByVal argument is not a pointer!")((A->getType()->isPointerTy() && "ByVal argument is not a pointer!"
) ? static_cast<void> (0) : __assert_fail ("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3318, __PRETTY_FUNCTION__))
;
3319 Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
3320 if (ArgOffset + Size > kParamTLSSize) break;
3321 const MaybeAlign ParamAlignment(CS.getParamAlignment(i));
3322 MaybeAlign Alignment = llvm::None;
3323 if (ParamAlignment)
3324 Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
3325 Value *AShadowPtr =
3326 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3327 /*isStore*/ false)
3328 .first;
3329
3330 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3331 Alignment, Size);
3332 // TODO(glider): need to copy origins.
3333 } else {
3334 Size = DL.getTypeAllocSize(A->getType());
3335 if (ArgOffset + Size > kParamTLSSize) break;
3336 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3337 kShadowTLSAlignment.value());
3338 Constant *Cst = dyn_cast<Constant>(ArgShadow);
3339 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
3340 }
3341 if (MS.TrackOrigins && !ArgIsInitialized)
3342 IRB.CreateStore(getOrigin(A),
3343 getOriginPtrForArgument(A, IRB, ArgOffset));
3344 (void)Store;
3345 assert(Size != 0 && Store != nullptr)((Size != 0 && Store != nullptr) ? static_cast<void
> (0) : __assert_fail ("Size != 0 && Store != nullptr"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3345, __PRETTY_FUNCTION__))
;
3346 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Param:" << *Store <<
"\n"; } } while (false)
;
3347 ArgOffset += alignTo(Size, 8);
3348 }
3349 LLVM_DEBUG(dbgs() << " done with call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " done with call args\n"; } } while
(false)
;
3350
3351 FunctionType *FT = CS.getFunctionType();
3352 if (FT->isVarArg()) {
3353 VAHelper->visitCallSite(CS, IRB);
3354 }
3355
3356 // Now, get the shadow for the RetVal.
3357 if (!I.getType()->isSized()) return;
3358 // Don't emit the epilogue for musttail call returns.
3359 if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return;
3360 IRBuilder<> IRBBefore(&I);
3361 // Until we have full dynamic coverage, make sure the retval shadow is 0.
3362 Value *Base = getShadowPtrForRetval(&I, IRBBefore);
3363 IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base,
3364 kShadowTLSAlignment.value());
3365 BasicBlock::iterator NextInsn;
3366 if (CS.isCall()) {
3367 NextInsn = ++I.getIterator();
3368 assert(NextInsn != I.getParent()->end())((NextInsn != I.getParent()->end()) ? static_cast<void>
(0) : __assert_fail ("NextInsn != I.getParent()->end()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3368, __PRETTY_FUNCTION__))
;
3369 } else {
3370 BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
3371 if (!NormalDest->getSinglePredecessor()) {
3372 // FIXME: this case is tricky, so we are just conservative here.
3373 // Perhaps we need to split the edge between this BB and NormalDest,
3374 // but a naive attempt to use SplitEdge leads to a crash.
3375 setShadow(&I, getCleanShadow(&I));
3376 setOrigin(&I, getCleanOrigin());
3377 return;
3378 }
3379 // FIXME: NextInsn is likely in a basic block that has not been visited yet.
3380 // Anything inserted there will be instrumented by MSan later!
3381 NextInsn = NormalDest->getFirstInsertionPt();
3382 assert(NextInsn != NormalDest->end() &&((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load"
) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3383, __PRETTY_FUNCTION__))
3383 "Could not find insertion point for retval shadow load")((NextInsn != NormalDest->end() && "Could not find insertion point for retval shadow load"
) ? static_cast<void> (0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3383, __PRETTY_FUNCTION__))
;
3384 }
3385 IRBuilder<> IRBAfter(&*NextInsn);
3386 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3387 getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter),
3388 kShadowTLSAlignment.value(), "_msret");
3389 setShadow(&I, RetvalShadow);
3390 if (MS.TrackOrigins)
3391 setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy,
3392 getOriginPtrForRetval(IRBAfter)));
3393 }
3394
3395 bool isAMustTailRetVal(Value *RetVal) {
3396 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3397 RetVal = I->getOperand(0);
3398 }
3399 if (auto *I = dyn_cast<CallInst>(RetVal)) {
3400 return I->isMustTailCall();
3401 }
3402 return false;
3403 }
3404
3405 void visitReturnInst(ReturnInst &I) {
3406 IRBuilder<> IRB(&I);
3407 Value *RetVal = I.getReturnValue();
3408 if (!RetVal) return;
3409 // Don't emit the epilogue for musttail call returns.
3410 if (isAMustTailRetVal(RetVal)) return;
3411 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3412 if (CheckReturnValue) {
3413 insertShadowCheck(RetVal, &I);
3414 Value *Shadow = getCleanShadow(RetVal);
3415 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment.value());
3416 } else {
3417 Value *Shadow = getShadow(RetVal);
3418 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment.value());
3419 if (MS.TrackOrigins)
3420 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3421 }
3422 }
3423
3424 void visitPHINode(PHINode &I) {
3425 IRBuilder<> IRB(&I);
3426 if (!PropagateShadow) {
3427 setShadow(&I, getCleanShadow(&I));
3428 setOrigin(&I, getCleanOrigin());
3429 return;
3430 }
3431
3432 ShadowPHINodes.push_back(&I);
3433 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3434 "_msphi_s"));
3435 if (MS.TrackOrigins)
3436 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3437 "_msphi_o"));
3438 }
3439
3440 Value *getLocalVarDescription(AllocaInst &I) {
3441 SmallString<2048> StackDescriptionStorage;
3442 raw_svector_ostream StackDescription(StackDescriptionStorage);
3443 // We create a string with a description of the stack allocation and
3444 // pass it into __msan_set_alloca_origin.
3445 // It will be printed by the run-time if stack-originated UMR is found.
3446 // The first 4 bytes of the string are set to '----' and will be replaced
3447 // by __msan_va_arg_overflow_size_tls at the first call.
3448 StackDescription << "----" << I.getName() << "@" << F.getName();
3449 return createPrivateNonConstGlobalForString(*F.getParent(),
3450 StackDescription.str());
3451 }
3452
3453 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3454 if (PoisonStack && ClPoisonStackWithCall) {
3455 IRB.CreateCall(MS.MsanPoisonStackFn,
3456 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3457 } else {
3458 Value *ShadowBase, *OriginBase;
3459 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
3460 &I, IRB, IRB.getInt8Ty(), Align::None(), /*isStore*/ true);
3461
3462 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3463 IRB.CreateMemSet(ShadowBase, PoisonValue, Len,
3464 MaybeAlign(I.getAlignment()));
3465 }
3466
3467 if (PoisonStack && MS.TrackOrigins) {
3468 Value *Descr = getLocalVarDescription(I);
3469 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3470 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3471 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3472 IRB.CreatePointerCast(&F, MS.IntptrTy)});
3473 }
3474 }
3475
3476 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3477 Value *Descr = getLocalVarDescription(I);
3478 if (PoisonStack) {
3479 IRB.CreateCall(MS.MsanPoisonAllocaFn,
3480 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3481 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3482 } else {
3483 IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
3484 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3485 }
3486 }
3487
3488 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
3489 if (!InsPoint)
3490 InsPoint = &I;
3491 IRBuilder<> IRB(InsPoint->getNextNode());
3492 const DataLayout &DL = F.getParent()->getDataLayout();
3493 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3494 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3495 if (I.isArrayAllocation())
3496 Len = IRB.CreateMul(Len, I.getArraySize());
3497
3498 if (MS.CompileKernel)
3499 poisonAllocaKmsan(I, IRB, Len);
3500 else
3501 poisonAllocaUserspace(I, IRB, Len);
3502 }
3503
3504 void visitAllocaInst(AllocaInst &I) {
3505 setShadow(&I, getCleanShadow(&I));
3506 setOrigin(&I, getCleanOrigin());
3507 // We'll get to this alloca later unless it's poisoned at the corresponding
3508 // llvm.lifetime.start.
3509 AllocaSet.insert(&I);
3510 }
3511
3512 void visitSelectInst(SelectInst& I) {
3513 IRBuilder<> IRB(&I);
3514 // a = select b, c, d
3515 Value *B = I.getCondition();
3516 Value *C = I.getTrueValue();
3517 Value *D = I.getFalseValue();
3518 Value *Sb = getShadow(B);
3519 Value *Sc = getShadow(C);
3520 Value *Sd = getShadow(D);
3521
3522 // Result shadow if condition shadow is 0.
3523 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3524 Value *Sa1;
3525 if (I.getType()->isAggregateType()) {
3526 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3527 // an extra "select". This results in much more compact IR.
3528 // Sa = select Sb, poisoned, (select b, Sc, Sd)
3529 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3530 } else {
3531 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3532 // If Sb (condition is poisoned), look for bits in c and d that are equal
3533 // and both unpoisoned.
3534 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3535
3536 // Cast arguments to shadow-compatible type.
3537 C = CreateAppToShadowCast(IRB, C);
3538 D = CreateAppToShadowCast(IRB, D);
3539
3540 // Result shadow if condition shadow is 1.
3541 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
3542 }
3543 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3544 setShadow(&I, Sa);
3545 if (MS.TrackOrigins) {
3546 // Origins are always i32, so any vector conditions must be flattened.
3547 // FIXME: consider tracking vector origins for app vectors?
3548 if (B->getType()->isVectorTy()) {
3549 Type *FlatTy = getShadowTyNoVec(B->getType());
3550 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3551 ConstantInt::getNullValue(FlatTy));
3552 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3553 ConstantInt::getNullValue(FlatTy));
3554 }
3555 // a = select b, c, d
3556 // Oa = Sb ? Ob : (b ? Oc : Od)
3557 setOrigin(
3558 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3559 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3560 getOrigin(I.getFalseValue()))));
3561 }
3562 }
3563
3564 void visitLandingPadInst(LandingPadInst &I) {
3565 // Do nothing.
3566 // See https://github.com/google/sanitizers/issues/504
3567 setShadow(&I, getCleanShadow(&I));
3568 setOrigin(&I, getCleanOrigin());
3569 }
3570
3571 void visitCatchSwitchInst(CatchSwitchInst &I) {
3572 setShadow(&I, getCleanShadow(&I));
3573 setOrigin(&I, getCleanOrigin());
3574 }
3575
3576 void visitFuncletPadInst(FuncletPadInst &I) {
3577 setShadow(&I, getCleanShadow(&I));
3578 setOrigin(&I, getCleanOrigin());
3579 }
3580
3581 void visitGetElementPtrInst(GetElementPtrInst &I) {
3582 handleShadowOr(I);
3583 }
3584
3585 void visitExtractValueInst(ExtractValueInst &I) {
3586 IRBuilder<> IRB(&I);
3587 Value *Agg = I.getAggregateOperand();
3588 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ExtractValue: " << I <<
"\n"; } } while (false)
;
3589 Value *AggShadow = getShadow(Agg);
3590 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
3591 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
3592 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ResShadow: " << *ResShadow
<< "\n"; } } while (false)
;
3593 setShadow(&I, ResShadow);
3594 setOriginForNaryOp(I);
3595 }
3596
3597 void visitInsertValueInst(InsertValueInst &I) {
3598 IRBuilder<> IRB(&I);
3599 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "InsertValue: " << I <<
"\n"; } } while (false)
;
3600 Value *AggShadow = getShadow(I.getAggregateOperand());
3601 Value *InsShadow = getShadow(I.getInsertedValueOperand());
3602 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
3603 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " InsShadow: " << *InsShadow
<< "\n"; } } while (false)
;
3604 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
3605 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Res: " << *Res <<
"\n"; } } while (false)
;
3606 setShadow(&I, Res);
3607 setOriginForNaryOp(I);
3608 }
3609
3610 void dumpInst(Instruction &I) {
3611 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
3612 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
3613 } else {
3614 errs() << "ZZZ " << I.getOpcodeName() << "\n";
3615 }
3616 errs() << "QQQ " << I << "\n";
3617 }
3618
3619 void visitResumeInst(ResumeInst &I) {
3620 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Resume: " << I << "\n"
; } } while (false)
;
3621 // Nothing to do here.
3622 }
3623
3624 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
3625 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CleanupReturn: " << CRI <<
"\n"; } } while (false)
;
3626 // Nothing to do here.
3627 }
3628
3629 void visitCatchReturnInst(CatchReturnInst &CRI) {
3630 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CatchReturn: " << CRI <<
"\n"; } } while (false)
;
3631 // Nothing to do here.
3632 }
3633
3634 void instrumentAsmArgument(Value *Operand, Instruction &I, IRBuilder<> &IRB,
3635 const DataLayout &DL, bool isOutput) {
3636 // For each assembly argument, we check its value for being initialized.
3637 // If the argument is a pointer, we assume it points to a single element
3638 // of the corresponding type (or to a 8-byte word, if the type is unsized).
3639 // Each such pointer is instrumented with a call to the runtime library.
3640 Type *OpType = Operand->getType();
3641 // Check the operand value itself.
3642 insertShadowCheck(Operand, &I);
3643 if (!OpType->isPointerTy() || !isOutput) {
3644 assert(!isOutput)((!isOutput) ? static_cast<void> (0) : __assert_fail ("!isOutput"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3644, __PRETTY_FUNCTION__))
;
3645 return;
3646 }
3647 Type *ElType = OpType->getPointerElementType();
3648 if (!ElType->isSized())
3649 return;
3650 int Size = DL.getTypeStoreSize(ElType);
3651 Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
3652 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
3653 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
3654 }
3655
3656 /// Get the number of output arguments returned by pointers.
3657 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
3658 int NumRetOutputs = 0;
3659 int NumOutputs = 0;
3660 Type *RetTy = cast<Value>(CB)->getType();
3661 if (!RetTy->isVoidTy()) {
3662 // Register outputs are returned via the CallInst return value.
3663 auto *ST = dyn_cast<StructType>(RetTy);
3664 if (ST)
3665 NumRetOutputs = ST->getNumElements();
3666 else
3667 NumRetOutputs = 1;
3668 }
3669 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
3670 for (size_t i = 0, n = Constraints.size(); i < n; i++) {
3671 InlineAsm::ConstraintInfo Info = Constraints[i];
3672 switch (Info.Type) {
3673 case InlineAsm::isOutput:
3674 NumOutputs++;
3675 break;
3676 default:
3677 break;
3678 }
3679 }
3680 return NumOutputs - NumRetOutputs;
3681 }
3682
3683 void visitAsmInstruction(Instruction &I) {
3684 // Conservative inline assembly handling: check for poisoned shadow of
3685 // asm() arguments, then unpoison the result and all the memory locations
3686 // pointed to by those arguments.
3687 // An inline asm() statement in C++ contains lists of input and output
3688 // arguments used by the assembly code. These are mapped to operands of the
3689 // CallInst as follows:
3690 // - nR register outputs ("=r) are returned by value in a single structure
3691 // (SSA value of the CallInst);
3692 // - nO other outputs ("=m" and others) are returned by pointer as first
3693 // nO operands of the CallInst;
3694 // - nI inputs ("r", "m" and others) are passed to CallInst as the
3695 // remaining nI operands.
3696 // The total number of asm() arguments in the source is nR+nO+nI, and the
3697 // corresponding CallInst has nO+nI+1 operands (the last operand is the
3698 // function to be called).
3699 const DataLayout &DL = F.getParent()->getDataLayout();
3700 CallBase *CB = cast<CallBase>(&I);
3701 IRBuilder<> IRB(&I);
3702 InlineAsm *IA = cast<InlineAsm>(CB->getCalledValue());
3703 int OutputArgs = getNumOutputArgs(IA, CB);
3704 // The last operand of a CallInst is the function itself.
3705 int NumOperands = CB->getNumOperands() - 1;
3706
3707 // Check input arguments. Doing so before unpoisoning output arguments, so
3708 // that we won't overwrite uninit values before checking them.
3709 for (int i = OutputArgs; i < NumOperands; i++) {
3710 Value *Operand = CB->getOperand(i);
3711 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false);
3712 }
3713 // Unpoison output arguments. This must happen before the actual InlineAsm
3714 // call, so that the shadow for memory published in the asm() statement
3715 // remains valid.
3716 for (int i = 0; i < OutputArgs; i++) {
3717 Value *Operand = CB->getOperand(i);
3718 instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true);
3719 }
3720
3721 setShadow(&I, getCleanShadow(&I));
3722 setOrigin(&I, getCleanOrigin());
3723 }
3724
3725 void visitInstruction(Instruction &I) {
3726 // Everything else: stop propagating and check for poisoned shadow.
3727 if (ClDumpStrictInstructions)
3728 dumpInst(I);
3729 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DEFAULT: " << I << "\n"
; } } while (false)
;
3730 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
3731 Value *Operand = I.getOperand(i);
3732 if (Operand->getType()->isSized())
3733 insertShadowCheck(Operand, &I);
3734 }
3735 setShadow(&I, getCleanShadow(&I));
3736 setOrigin(&I, getCleanOrigin());
3737 }
3738};
3739
3740/// AMD64-specific implementation of VarArgHelper.
3741struct VarArgAMD64Helper : public VarArgHelper {
3742 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
3743 // See a comment in visitCallSite for more details.
3744 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
3745 static const unsigned AMD64FpEndOffsetSSE = 176;
3746 // If SSE is disabled, fp_offset in va_list is zero.
3747 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
3748
3749 unsigned AMD64FpEndOffset;
3750 Function &F;
3751 MemorySanitizer &MS;
3752 MemorySanitizerVisitor &MSV;
3753 Value *VAArgTLSCopy = nullptr;
3754 Value *VAArgTLSOriginCopy = nullptr;
3755 Value *VAArgOverflowSize = nullptr;
3756
3757 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3758
3759 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3760
3761 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
3762 MemorySanitizerVisitor &MSV)
3763 : F(F), MS(MS), MSV(MSV) {
3764 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
3765 for (const auto &Attr : F.getAttributes().getFnAttributes()) {
3766 if (Attr.isStringAttribute() &&
3767 (Attr.getKindAsString() == "target-features")) {
3768 if (Attr.getValueAsString().contains("-sse"))
3769 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
3770 break;
3771 }
3772 }
3773 }
3774
3775 ArgKind classifyArgument(Value* arg) {
3776 // A very rough approximation of X86_64 argument classification rules.
3777 Type *T = arg->getType();
3778 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
3779 return AK_FloatingPoint;
3780 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3781 return AK_GeneralPurpose;
3782 if (T->isPointerTy())
3783 return AK_GeneralPurpose;
3784 return AK_Memory;
3785 }
3786
3787 // For VarArg functions, store the argument shadow in an ABI-specific format
3788 // that corresponds to va_list layout.
3789 // We do this because Clang lowers va_arg in the frontend, and this pass
3790 // only sees the low level code that deals with va_list internals.
3791 // A much easier alternative (provided that Clang emits va_arg instructions)
3792 // would have been to associate each live instance of va_list with a copy of
3793 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
3794 // order.
3795 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3796 unsigned GpOffset = 0;
3797 unsigned FpOffset = AMD64GpEndOffset;
3798 unsigned OverflowOffset = AMD64FpEndOffset;
3799 const DataLayout &DL = F.getParent()->getDataLayout();
3800 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3801 ArgIt != End; ++ArgIt) {
3802 Value *A = *ArgIt;
3803 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3804 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3805 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3806 if (IsByVal) {
3807 // ByVal arguments always go to the overflow area.
3808 // Fixed arguments passed through the overflow area will be stepped
3809 // over by va_start, so don't count them towards the offset.
3810 if (IsFixed)
3811 continue;
3812 assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void>
(0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3812, __PRETTY_FUNCTION__))
;
3813 Type *RealTy = A->getType()->getPointerElementType();
3814 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3815 Value *ShadowBase = getShadowPtrForVAArgument(
3816 RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
3817 Value *OriginBase = nullptr;
3818 if (MS.TrackOrigins)
3819 OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
3820 OverflowOffset += alignTo(ArgSize, 8);
3821 if (!ShadowBase)
3822 continue;
3823 Value *ShadowPtr, *OriginPtr;
3824 std::tie(ShadowPtr, OriginPtr) =
3825 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
3826 /*isStore*/ false);
3827
3828 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
3829 kShadowTLSAlignment, ArgSize);
3830 if (MS.TrackOrigins)
3831 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
3832 kShadowTLSAlignment, ArgSize);
3833 } else {
3834 ArgKind AK = classifyArgument(A);
3835 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
3836 AK = AK_Memory;
3837 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
3838 AK = AK_Memory;
3839 Value *ShadowBase, *OriginBase = nullptr;
3840 switch (AK) {
3841 case AK_GeneralPurpose:
3842 ShadowBase =
3843 getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
3844 if (MS.TrackOrigins)
3845 OriginBase =
3846 getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
3847 GpOffset += 8;
3848 break;
3849 case AK_FloatingPoint:
3850 ShadowBase =
3851 getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
3852 if (MS.TrackOrigins)
3853 OriginBase =
3854 getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
3855 FpOffset += 16;
3856 break;
3857 case AK_Memory:
3858 if (IsFixed)
3859 continue;
3860 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3861 ShadowBase =
3862 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
3863 if (MS.TrackOrigins)
3864 OriginBase =
3865 getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3866 OverflowOffset += alignTo(ArgSize, 8);
3867 }
3868 // Take fixed arguments into account for GpOffset and FpOffset,
3869 // but don't actually store shadows for them.
3870 // TODO(glider): don't call get*PtrForVAArgument() for them.
3871 if (IsFixed)
3872 continue;
3873 if (!ShadowBase)
3874 continue;
3875 Value *Shadow = MSV.getShadow(A);
3876 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment.value());
3877 if (MS.TrackOrigins) {
3878 Value *Origin = MSV.getOrigin(A);
3879 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
3880 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
3881 std::max(kShadowTLSAlignment, kMinOriginAlignment));
3882 }
3883 }
3884 }
3885 Constant *OverflowSize =
3886 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
3887 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3888 }
3889
3890 /// Compute the shadow address for a given va_arg.
3891 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3892 unsigned ArgOffset, unsigned ArgSize) {
3893 // Make sure we don't overflow __msan_va_arg_tls.
3894 if (ArgOffset + ArgSize > kParamTLSSize)
3895 return nullptr;
3896 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3897 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3898 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3899 "_msarg_va_s");
3900 }
3901
3902 /// Compute the origin address for a given va_arg.
3903 Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
3904 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
3905 // getOriginPtrForVAArgument() is always called after
3906 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
3907 // overflow.
3908 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3909 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
3910 "_msarg_va_o");
3911 }
3912
3913 void unpoisonVAListTagForInst(IntrinsicInst &I) {
3914 IRBuilder<> IRB(&I);
3915 Value *VAListTag = I.getArgOperand(0);
3916 Value *ShadowPtr, *OriginPtr;
3917 const Align Alignment = Align(8);
3918 std::tie(ShadowPtr, OriginPtr) =
3919 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
3920 /*isStore*/ true);
3921
3922 // Unpoison the whole __va_list_tag.
3923 // FIXME: magic ABI constants.
3924 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3925 /* size */ 24, Alignment, false);
3926 // We shouldn't need to zero out the origins, as they're only checked for
3927 // nonzero shadow.
3928 }
3929
3930 void visitVAStartInst(VAStartInst &I) override {
3931 if (F.getCallingConv() == CallingConv::Win64)
3932 return;
3933 VAStartInstrumentationList.push_back(&I);
3934 unpoisonVAListTagForInst(I);
3935 }
3936
3937 void visitVACopyInst(VACopyInst &I) override {
3938 if (F.getCallingConv() == CallingConv::Win64) return;
3939 unpoisonVAListTagForInst(I);
3940 }
3941
3942 void finalizeInstrumentation() override {
3943 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3944, __PRETTY_FUNCTION__))
3944 "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3944, __PRETTY_FUNCTION__))
;
3945 if (!VAStartInstrumentationList.empty()) {
3946 // If there is a va_start in this function, make a backup copy of
3947 // va_arg_tls somewhere in the function entry block.
3948 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3949 VAArgOverflowSize =
3950 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
3951 Value *CopySize =
3952 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3953 VAArgOverflowSize);
3954 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3955 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
3956 if (MS.TrackOrigins) {
3957 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3958 IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
3959 Align(8), CopySize);
3960 }
3961 }
3962
3963 // Instrument va_start.
3964 // Copy va_list shadow from the backup copy of the TLS contents.
3965 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3966 CallInst *OrigInst = VAStartInstrumentationList[i];
3967 IRBuilder<> IRB(OrigInst->getNextNode());
3968 Value *VAListTag = OrigInst->getArgOperand(0);
3969
3970 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
3971 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
3972 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3973 ConstantInt::get(MS.IntptrTy, 16)),
3974 PointerType::get(RegSaveAreaPtrTy, 0));
3975 Value *RegSaveAreaPtr =
3976 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
3977 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3978 const Align Alignment = Align(16);
3979 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3980 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3981 Alignment, /*isStore*/ true);
3982 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3983 AMD64FpEndOffset);
3984 if (MS.TrackOrigins)
3985 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
3986 Alignment, AMD64FpEndOffset);
3987 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
3988 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
3989 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3990 ConstantInt::get(MS.IntptrTy, 8)),
3991 PointerType::get(OverflowArgAreaPtrTy, 0));
3992 Value *OverflowArgAreaPtr =
3993 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
3994 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
3995 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
3996 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
3997 Alignment, /*isStore*/ true);
3998 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3999 AMD64FpEndOffset);
4000 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4001 VAArgOverflowSize);
4002 if (MS.TrackOrigins) {
4003 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
4004 AMD64FpEndOffset);
4005 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4006 VAArgOverflowSize);
4007 }
4008 }
4009 }
4010};
4011
4012/// MIPS64-specific implementation of VarArgHelper.
4013struct VarArgMIPS64Helper : public VarArgHelper {
4014 Function &F;
4015 MemorySanitizer &MS;
4016 MemorySanitizerVisitor &MSV;
4017 Value *VAArgTLSCopy = nullptr;
4018 Value *VAArgSize = nullptr;
4019
4020 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4021
4022 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4023 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4024
4025 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4026 unsigned VAArgOffset = 0;
4027 const DataLayout &DL = F.getParent()->getDataLayout();
4028 for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
4029 CS.getFunctionType()->getNumParams(), End = CS.arg_end();
4030 ArgIt != End; ++ArgIt) {
4031 Triple TargetTriple(F.getParent()->getTargetTriple());
4032 Value *A = *ArgIt;
4033 Value *Base;
4034 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4035 if (TargetTriple.getArch() == Triple::mips64) {
4036 // Adjusting the shadow for argument with size < 8 to match the placement
4037 // of bits in big endian system
4038 if (ArgSize < 8)
4039 VAArgOffset += (8 - ArgSize);
4040 }
4041 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4042 VAArgOffset += ArgSize;
4043 VAArgOffset = alignTo(VAArgOffset, 8);
4044 if (!Base)
4045 continue;
4046 IRB.CreateAlignedStore(MSV.getShadow(A), Base,
4047 kShadowTLSAlignment.value());
4048 }
4049
4050 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4051 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4052 // a new class member i.e. it is the total size of all VarArgs.
4053 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4054 }
4055
4056 /// Compute the shadow address for a given va_arg.
4057 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4058 unsigned ArgOffset, unsigned ArgSize) {
4059 // Make sure we don't overflow __msan_va_arg_tls.
4060 if (ArgOffset + ArgSize > kParamTLSSize)
4061 return nullptr;
4062 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4063 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4064 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4065 "_msarg");
4066 }
4067
4068 void visitVAStartInst(VAStartInst &I) override {
4069 IRBuilder<> IRB(&I);
4070 VAStartInstrumentationList.push_back(&I);
4071 Value *VAListTag = I.getArgOperand(0);
4072 Value *ShadowPtr, *OriginPtr;
4073 const Align Alignment = Align(8);
4074 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4075 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4076 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4077 /* size */ 8, Alignment, false);
4078 }
4079
4080 void visitVACopyInst(VACopyInst &I) override {
4081 IRBuilder<> IRB(&I);
4082 VAStartInstrumentationList.push_back(&I);
4083 Value *VAListTag = I.getArgOperand(0);
4084 Value *ShadowPtr, *OriginPtr;
4085 const Align Alignment = Align(8);
4086 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4087 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4088 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4089 /* size */ 8, Alignment, false);
4090 }
4091
4092 void finalizeInstrumentation() override {
4093 assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4094, __PRETTY_FUNCTION__))
4094 "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4094, __PRETTY_FUNCTION__))
;
4095 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4096 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4097 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4098 VAArgSize);
4099
4100 if (!VAStartInstrumentationList.empty()) {
4101 // If there is a va_start in this function, make a backup copy of
4102 // va_arg_tls somewhere in the function entry block.
4103 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4104 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4105 }
4106
4107 // Instrument va_start.
4108 // Copy va_list shadow from the backup copy of the TLS contents.
4109 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4110 CallInst *OrigInst = VAStartInstrumentationList[i];
4111 IRBuilder<> IRB(OrigInst->getNextNode());
4112 Value *VAListTag = OrigInst->getArgOperand(0);
4113 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4114 Value *RegSaveAreaPtrPtr =
4115 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4116 PointerType::get(RegSaveAreaPtrTy, 0));
4117 Value *RegSaveAreaPtr =
4118 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4119 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4120 const Align Alignment = Align(8);
4121 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4122 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4123 Alignment, /*isStore*/ true);
4124 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4125 CopySize);
4126 }
4127 }
4128};
4129
4130/// AArch64-specific implementation of VarArgHelper.
4131struct VarArgAArch64Helper : public VarArgHelper {
4132 static const unsigned kAArch64GrArgSize = 64;
4133 static const unsigned kAArch64VrArgSize = 128;
4134
4135 static const unsigned AArch64GrBegOffset = 0;
4136 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4137 // Make VR space aligned to 16 bytes.
4138 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4139 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
4140 + kAArch64VrArgSize;
4141 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4142
4143 Function &F;
4144 MemorySanitizer &MS;
4145 MemorySanitizerVisitor &MSV;
4146 Value *VAArgTLSCopy = nullptr;
4147 Value *VAArgOverflowSize = nullptr;
4148
4149 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4150
4151 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4152
4153 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4154 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4155
4156 ArgKind classifyArgument(Value* arg) {
4157 Type *T = arg->getType();
4158 if (T->isFPOrFPVectorTy())
4159 return AK_FloatingPoint;
4160 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4161 || (T->isPointerTy()))
4162 return AK_GeneralPurpose;
4163 return AK_Memory;
4164 }
4165
4166 // The instrumentation stores the argument shadow in a non ABI-specific
4167 // format because it does not know which argument is named (since Clang,
4168 // like x86_64 case, lowers the va_args in the frontend and this pass only
4169 // sees the low level code that deals with va_list internals).
4170 // The first seven GR registers are saved in the first 56 bytes of the
4171 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4172 // the remaining arguments.
4173 // Using constant offset within the va_arg TLS array allows fast copy
4174 // in the finalize instrumentation.
4175 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4176 unsigned GrOffset = AArch64GrBegOffset;
4177 unsigned VrOffset = AArch64VrBegOffset;
4178 unsigned OverflowOffset = AArch64VAEndOffset;
4179
4180 const DataLayout &DL = F.getParent()->getDataLayout();
4181 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
4182 ArgIt != End; ++ArgIt) {
4183 Value *A = *ArgIt;
4184 unsigned ArgNo = CS.getArgumentNo(ArgIt);
4185 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
4186 ArgKind AK = classifyArgument(A);
4187 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4188 AK = AK_Memory;
4189 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
4190 AK = AK_Memory;
4191 Value *Base;
4192 switch (AK) {
4193 case AK_GeneralPurpose:
4194 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4195 GrOffset += 8;
4196 break;
4197 case AK_FloatingPoint:
4198 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4199 VrOffset += 16;
4200 break;
4201 case AK_Memory:
4202 // Don't count fixed arguments in the overflow area - va_start will
4203 // skip right over them.
4204 if (IsFixed)
4205 continue;
4206 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4207 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4208 alignTo(ArgSize, 8));
4209 OverflowOffset += alignTo(ArgSize, 8);
4210 break;
4211 }
4212 // Count Gp/Vr fixed arguments to their respective offsets, but don't
4213 // bother to actually store a shadow.
4214 if (IsFixed)
4215 continue;
4216 if (!Base)
4217 continue;
4218 IRB.CreateAlignedStore(MSV.getShadow(A), Base,
4219 kShadowTLSAlignment.value());
4220 }
4221 Constant *OverflowSize =
4222 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4223 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4224 }
4225
4226 /// Compute the shadow address for a given va_arg.
4227 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4228 unsigned ArgOffset, unsigned ArgSize) {
4229 // Make sure we don't overflow __msan_va_arg_tls.
4230 if (ArgOffset + ArgSize > kParamTLSSize)
4231 return nullptr;
4232 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4233 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4234 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4235 "_msarg");
4236 }
4237
4238 void visitVAStartInst(VAStartInst &I) override {
4239 IRBuilder<> IRB(&I);
4240 VAStartInstrumentationList.push_back(&I);
4241 Value *VAListTag = I.getArgOperand(0);
4242 Value *ShadowPtr, *OriginPtr;
4243 const Align Alignment = Align(8);
4244 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4245 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4246 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4247 /* size */ 32, Alignment, false);
4248 }
4249
4250 void visitVACopyInst(VACopyInst &I) override {
4251 IRBuilder<> IRB(&I);
4252 VAStartInstrumentationList.push_back(&I);
4253 Value *VAListTag = I.getArgOperand(0);
4254 Value *ShadowPtr, *OriginPtr;
4255 const Align Alignment = Align(8);
4256 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4257 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4258 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4259 /* size */ 32, Alignment, false);
4260 }
4261
4262 // Retrieve a va_list field of 'void*' size.
4263 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4264 Value *SaveAreaPtrPtr =
4265 IRB.CreateIntToPtr(
4266 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4267 ConstantInt::get(MS.IntptrTy, offset)),
4268 Type::getInt64PtrTy(*MS.C));
4269 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4270 }
4271
4272 // Retrieve a va_list field of 'int' size.
4273 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4274 Value *SaveAreaPtr =
4275 IRB.CreateIntToPtr(
4276 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4277 ConstantInt::get(MS.IntptrTy, offset)),
4278 Type::getInt32PtrTy(*MS.C));
4279 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4280 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4281 }
4282
4283 void finalizeInstrumentation() override {
4284 assert(!VAArgOverflowSize && !VAArgTLSCopy &&((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4285, __PRETTY_FUNCTION__))
4285 "finalizeInstrumentation called twice")((!VAArgOverflowSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4285, __PRETTY_FUNCTION__))
;
4286 if (!VAStartInstrumentationList.empty()) {
4287 // If there is a va_start in this function, make a backup copy of
4288 // va_arg_tls somewhere in the function entry block.
4289 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4290 VAArgOverflowSize =
4291 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4292 Value *CopySize =
4293 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
4294 VAArgOverflowSize);
4295 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4296 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4297 }
4298
4299 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4300 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4301
4302 // Instrument va_start, copy va_list shadow from the backup copy of
4303 // the TLS contents.
4304 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4305 CallInst *OrigInst = VAStartInstrumentationList[i];
4306 IRBuilder<> IRB(OrigInst->getNextNode());
4307
4308 Value *VAListTag = OrigInst->getArgOperand(0);
4309
4310 // The variadic ABI for AArch64 creates two areas to save the incoming
4311 // argument registers (one for 64-bit general register xn-x7 and another
4312 // for 128-bit FP/SIMD vn-v7).
4313 // We need then to propagate the shadow arguments on both regions
4314 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4315 // The remaning arguments are saved on shadow for 'va::stack'.
4316 // One caveat is it requires only to propagate the non-named arguments,
4317 // however on the call site instrumentation 'all' the arguments are
4318 // saved. So to copy the shadow values from the va_arg TLS array
4319 // we need to adjust the offset for both GR and VR fields based on
4320 // the __{gr,vr}_offs value (since they are stores based on incoming
4321 // named arguments).
4322
4323 // Read the stack pointer from the va_list.
4324 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4325
4326 // Read both the __gr_top and __gr_off and add them up.
4327 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4328 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4329
4330 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4331
4332 // Read both the __vr_top and __vr_off and add them up.
4333 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4334 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4335
4336 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4337
4338 // It does not know how many named arguments is being used and, on the
4339 // callsite all the arguments were saved. Since __gr_off is defined as
4340 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4341 // argument by ignoring the bytes of shadow from named arguments.
4342 Value *GrRegSaveAreaShadowPtrOff =
4343 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4344
4345 Value *GrRegSaveAreaShadowPtr =
4346 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4347 Align(8), /*isStore*/ true)
4348 .first;
4349
4350 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4351 GrRegSaveAreaShadowPtrOff);
4352 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4353
4354 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
4355 GrCopySize);
4356
4357 // Again, but for FP/SIMD values.
4358 Value *VrRegSaveAreaShadowPtrOff =
4359 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4360
4361 Value *VrRegSaveAreaShadowPtr =
4362 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4363 Align(8), /*isStore*/ true)
4364 .first;
4365
4366 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4367 IRB.getInt8Ty(),
4368 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4369 IRB.getInt32(AArch64VrBegOffset)),
4370 VrRegSaveAreaShadowPtrOff);
4371 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4372
4373 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
4374 VrCopySize);
4375
4376 // And finally for remaining arguments.
4377 Value *StackSaveAreaShadowPtr =
4378 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4379 Align(16), /*isStore*/ true)
4380 .first;
4381
4382 Value *StackSrcPtr =
4383 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4384 IRB.getInt32(AArch64VAEndOffset));
4385
4386 IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
4387 Align(16), VAArgOverflowSize);
4388 }
4389 }
4390};
4391
4392/// PowerPC64-specific implementation of VarArgHelper.
4393struct VarArgPowerPC64Helper : public VarArgHelper {
4394 Function &F;
4395 MemorySanitizer &MS;
4396 MemorySanitizerVisitor &MSV;
4397 Value *VAArgTLSCopy = nullptr;
4398 Value *VAArgSize = nullptr;
4399
4400 SmallVector<CallInst*, 16> VAStartInstrumentationList;
4401
4402 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4403 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4404
4405 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
4406 // For PowerPC, we need to deal with alignment of stack arguments -
4407 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4408 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4409 // and QPX vectors are aligned to 32 bytes. For that reason, we
4410 // compute current offset from stack pointer (which is always properly
4411 // aligned), and offset for the first vararg, then subtract them.
4412 unsigned VAArgBase;
4413 Triple TargetTriple(F.getParent()->getTargetTriple());
4414 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4415 // and 32 bytes for ABIv2. This is usually determined by target
4416 // endianness, but in theory could be overriden by function attribute.
4417 // For simplicity, we ignore it here (it'd only matter for QPX vectors).
4418 if (TargetTriple.getArch() == Triple::ppc64)
4419 VAArgBase = 48;
4420 else
4421 VAArgBase = 32;
4422 unsigned VAArgOffset = VAArgBase;
4423 const DataLayout &DL = F.getParent()->getDataLayout();
4424 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
4425 ArgIt != End; ++ArgIt) {
4426 Value *A = *ArgIt;
4427 unsigned ArgNo = CS.getArgumentNo(ArgIt);
4428 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
4429 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
4430 if (IsByVal) {
4431 assert(A->getType()->isPointerTy())((A->getType()->isPointerTy()) ? static_cast<void>
(0) : __assert_fail ("A->getType()->isPointerTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4431, __PRETTY_FUNCTION__))
;
4432 Type *RealTy = A->getType()->getPointerElementType();
4433 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4434 uint64_t ArgAlign = CS.getParamAlignment(ArgNo);
4435 if (ArgAlign < 8)
4436 ArgAlign = 8;
4437 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4438 if (!IsFixed) {
4439 Value *Base = getShadowPtrForVAArgument(
4440 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4441 if (Base) {
4442 Value *AShadowPtr, *AOriginPtr;
4443 std::tie(AShadowPtr, AOriginPtr) =
4444 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4445 kShadowTLSAlignment, /*isStore*/ false);
4446
4447 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4448 kShadowTLSAlignment, ArgSize);
4449 }
4450 }
4451 VAArgOffset += alignTo(ArgSize, 8);
4452 } else {
4453 Value *Base;
4454 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4455 uint64_t ArgAlign = 8;
4456 if (A->getType()->isArrayTy()) {
4457 // Arrays are aligned to element size, except for long double
4458 // arrays, which are aligned to 8 bytes.
4459 Type *ElementTy = A->getType()->getArrayElementType();
4460 if (!ElementTy->isPPC_FP128Ty())
4461 ArgAlign = DL.getTypeAllocSize(ElementTy);
4462 } else if (A->getType()->isVectorTy()) {
4463 // Vectors are naturally aligned.
4464 ArgAlign = DL.getTypeAllocSize(A->getType());
4465 }
4466 if (ArgAlign < 8)
4467 ArgAlign = 8;
4468 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4469 if (DL.isBigEndian()) {
4470 // Adjusting the shadow for argument with size < 8 to match the placement
4471 // of bits in big endian system
4472 if (ArgSize < 8)
4473 VAArgOffset += (8 - ArgSize);
4474 }
4475 if (!IsFixed) {
4476 Base = getShadowPtrForVAArgument(A->getType(), IRB,
4477 VAArgOffset - VAArgBase, ArgSize);
4478 if (Base)
4479 IRB.CreateAlignedStore(MSV.getShadow(A), Base,
4480 kShadowTLSAlignment.value());
4481 }
4482 VAArgOffset += ArgSize;
4483 VAArgOffset = alignTo(VAArgOffset, 8);
4484 }
4485 if (IsFixed)
4486 VAArgBase = VAArgOffset;
4487 }
4488
4489 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
4490 VAArgOffset - VAArgBase);
4491 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4492 // a new class member i.e. it is the total size of all VarArgs.
4493 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4494 }
4495
4496 /// Compute the shadow address for a given va_arg.
4497 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4498 unsigned ArgOffset, unsigned ArgSize) {
4499 // Make sure we don't overflow __msan_va_arg_tls.
4500 if (ArgOffset + ArgSize > kParamTLSSize)
4501 return nullptr;
4502 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4503 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4504 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4505 "_msarg");
4506 }
4507
4508 void visitVAStartInst(VAStartInst &I) override {
4509 IRBuilder<> IRB(&I);
4510 VAStartInstrumentationList.push_back(&I);
4511 Value *VAListTag = I.getArgOperand(0);
4512 Value *ShadowPtr, *OriginPtr;
4513 const Align Alignment = Align(8);
4514 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4515 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4516 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4517 /* size */ 8, Alignment, false);
4518 }
4519
4520 void visitVACopyInst(VACopyInst &I) override {
4521 IRBuilder<> IRB(&I);
4522 Value *VAListTag = I.getArgOperand(0);
4523 Value *ShadowPtr, *OriginPtr;
4524 const Align Alignment = Align(8);
4525 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4526 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4527 // Unpoison the whole __va_list_tag.
4528 // FIXME: magic ABI constants.
4529 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4530 /* size */ 8, Alignment, false);
4531 }
4532
4533 void finalizeInstrumentation() override {
4534 assert(!VAArgSize && !VAArgTLSCopy &&((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4535, __PRETTY_FUNCTION__))
4535 "finalizeInstrumentation called twice")((!VAArgSize && !VAArgTLSCopy && "finalizeInstrumentation called twice"
) ? static_cast<void> (0) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4535, __PRETTY_FUNCTION__))
;
4536 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
4537 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4538 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4539 VAArgSize);
4540
4541 if (!VAStartInstrumentationList.empty()) {
4542 // If there is a va_start in this function, make a backup copy of
4543 // va_arg_tls somewhere in the function entry block.
4544 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4545 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4546 }
4547
4548 // Instrument va_start.
4549 // Copy va_list shadow from the backup copy of the TLS contents.
4550 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4551 CallInst *OrigInst = VAStartInstrumentationList[i];
4552 IRBuilder<> IRB(OrigInst->getNextNode());
4553 Value *VAListTag = OrigInst->getArgOperand(0);
4554 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4555 Value *RegSaveAreaPtrPtr =
4556 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4557 PointerType::get(RegSaveAreaPtrTy, 0));
4558 Value *RegSaveAreaPtr =
4559 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4560 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4561 const Align Alignment = Align(8);
4562 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4563 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4564 Alignment, /*isStore*/ true);
4565 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4566 CopySize);
4567 }
4568 }
4569};
4570
4571/// A no-op implementation of VarArgHelper.
4572struct VarArgNoOpHelper : public VarArgHelper {
4573 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
4574 MemorySanitizerVisitor &MSV) {}
4575
4576 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
4577
4578 void visitVAStartInst(VAStartInst &I) override {}
4579
4580 void visitVACopyInst(VACopyInst &I) override {}
4581
4582 void finalizeInstrumentation() override {}
4583};
4584
4585} // end anonymous namespace
4586
4587static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
4588 MemorySanitizerVisitor &Visitor) {
4589 // VarArg handling is only implemented on AMD64. False positives are possible
4590 // on other platforms.
4591 Triple TargetTriple(Func.getParent()->getTargetTriple());
4592 if (TargetTriple.getArch() == Triple::x86_64)
4593 return new VarArgAMD64Helper(Func, Msan, Visitor);
4594 else if (TargetTriple.isMIPS64())
4595 return new VarArgMIPS64Helper(Func, Msan, Visitor);
4596 else if (TargetTriple.getArch() == Triple::aarch64)
4597 return new VarArgAArch64Helper(Func, Msan, Visitor);
4598 else if (TargetTriple.getArch() == Triple::ppc64 ||
4599 TargetTriple.getArch() == Triple::ppc64le)
4600 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
4601 else
4602 return new VarArgNoOpHelper(Func, Msan, Visitor);
4603}
4604
4605bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
4606 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
4607 return false;
4608
4609 MemorySanitizerVisitor Visitor(F, *this, TLI);
4610
4611 // Clear out readonly/readnone attributes.
4612 AttrBuilder B;
4613 B.addAttribute(Attribute::ReadOnly)
4614 .addAttribute(Attribute::ReadNone)
4615 .addAttribute(Attribute::WriteOnly)
4616 .addAttribute(Attribute::ArgMemOnly)
4617 .addAttribute(Attribute::Speculatable);
4618 F.removeAttributes(AttributeList::FunctionIndex, B);
4619
4620 return Visitor.runOnFunction();
4621}

/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h

1//===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the Type class. For more "Type"
10// stuff, look in DerivedTypes.h.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_TYPE_H
15#define LLVM_IR_TYPE_H
16
17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/Support/CBindingWrapping.h"
21#include "llvm/Support/Casting.h"
22#include "llvm/Support/Compiler.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/TypeSize.h"
25#include <cassert>
26#include <cstdint>
27#include <iterator>
28
29namespace llvm {
30
31template<class GraphType> struct GraphTraits;
32class IntegerType;
33class LLVMContext;
34class PointerType;
35class raw_ostream;
36class StringRef;
37
38/// The instances of the Type class are immutable: once they are created,
39/// they are never changed. Also note that only one instance of a particular
40/// type is ever created. Thus seeing if two types are equal is a matter of
41/// doing a trivial pointer comparison. To enforce that no two equal instances
42/// are created, Type instances can only be created via static factory methods
43/// in class Type and in derived classes. Once allocated, Types are never
44/// free'd.
45///
46class Type {
47public:
48 //===--------------------------------------------------------------------===//
49 /// Definitions of all of the base types for the Type system. Based on this
50 /// value, you can cast to a class defined in DerivedTypes.h.
51 /// Note: If you add an element to this, you need to add an element to the
52 /// Type::getPrimitiveType function, or else things will break!
53 /// Also update LLVMTypeKind and LLVMGetTypeKind () in the C binding.
54 ///
55 enum TypeID {
56 // PrimitiveTypes - make sure LastPrimitiveTyID stays up to date.
57 VoidTyID = 0, ///< 0: type with no size
58 HalfTyID, ///< 1: 16-bit floating point type
59 FloatTyID, ///< 2: 32-bit floating point type
60 DoubleTyID, ///< 3: 64-bit floating point type
61 X86_FP80TyID, ///< 4: 80-bit floating point type (X87)
62 FP128TyID, ///< 5: 128-bit floating point type (112-bit mantissa)
63 PPC_FP128TyID, ///< 6: 128-bit floating point type (two 64-bits, PowerPC)
64 LabelTyID, ///< 7: Labels
65 MetadataTyID, ///< 8: Metadata
66 X86_MMXTyID, ///< 9: MMX vectors (64 bits, X86 specific)
67 TokenTyID, ///< 10: Tokens
68
69 // Derived types... see DerivedTypes.h file.
70 // Make sure FirstDerivedTyID stays up to date!
71 IntegerTyID, ///< 11: Arbitrary bit width integers
72 FunctionTyID, ///< 12: Functions
73 StructTyID, ///< 13: Structures
74 ArrayTyID, ///< 14: Arrays
75 PointerTyID, ///< 15: Pointers
76 VectorTyID ///< 16: SIMD 'packed' format, or other vector type
77 };
78
79private:
80 /// This refers to the LLVMContext in which this type was uniqued.
81 LLVMContext &Context;
82
83 TypeID ID : 8; // The current base type of this type.
84 unsigned SubclassData : 24; // Space for subclasses to store data.
85 // Note that this should be synchronized with
86 // MAX_INT_BITS value in IntegerType class.
87
88protected:
89 friend class LLVMContextImpl;
90
91 explicit Type(LLVMContext &C, TypeID tid)
92 : Context(C), ID(tid), SubclassData(0) {}
93 ~Type() = default;
94
95 unsigned getSubclassData() const { return SubclassData; }
96
97 void setSubclassData(unsigned val) {
98 SubclassData = val;
99 // Ensure we don't have any accidental truncation.
100 assert(getSubclassData() == val && "Subclass data too large for field")((getSubclassData() == val && "Subclass data too large for field"
) ? static_cast<void> (0) : __assert_fail ("getSubclassData() == val && \"Subclass data too large for field\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 100, __PRETTY_FUNCTION__))
;
101 }
102
103 /// Keeps track of how many Type*'s there are in the ContainedTys list.
104 unsigned NumContainedTys = 0;
105
106 /// A pointer to the array of Types contained by this Type. For example, this
107 /// includes the arguments of a function type, the elements of a structure,
108 /// the pointee of a pointer, the element type of an array, etc. This pointer
109 /// may be 0 for types that don't contain other types (Integer, Double,
110 /// Float).
111 Type * const *ContainedTys = nullptr;
112
113 static bool isSequentialType(TypeID TyID) {
114 return TyID == ArrayTyID || TyID == VectorTyID;
115 }
116
117public:
118 /// Print the current type.
119 /// Omit the type details if \p NoDetails == true.
120 /// E.g., let %st = type { i32, i16 }
121 /// When \p NoDetails is true, we only print %st.
122 /// Put differently, \p NoDetails prints the type as if
123 /// inlined with the operands when printing an instruction.
124 void print(raw_ostream &O, bool IsForDebug = false,
125 bool NoDetails = false) const;
126
127 void dump() const;
128
129 /// Return the LLVMContext in which this type was uniqued.
130 LLVMContext &getContext() const { return Context; }
131
132 //===--------------------------------------------------------------------===//
133 // Accessors for working with types.
134 //
135
136 /// Return the type id for the type. This will return one of the TypeID enum
137 /// elements defined above.
138 TypeID getTypeID() const { return ID; }
139
140 /// Return true if this is 'void'.
141 bool isVoidTy() const { return getTypeID() == VoidTyID; }
142
143 /// Return true if this is 'half', a 16-bit IEEE fp type.
144 bool isHalfTy() const { return getTypeID() == HalfTyID; }
145
146 /// Return true if this is 'float', a 32-bit IEEE fp type.
147 bool isFloatTy() const { return getTypeID() == FloatTyID; }
148
149 /// Return true if this is 'double', a 64-bit IEEE fp type.
150 bool isDoubleTy() const { return getTypeID() == DoubleTyID; }
151
152 /// Return true if this is x86 long double.
153 bool isX86_FP80Ty() const { return getTypeID() == X86_FP80TyID; }
154
155 /// Return true if this is 'fp128'.
156 bool isFP128Ty() const { return getTypeID() == FP128TyID; }
157
158 /// Return true if this is powerpc long double.
159 bool isPPC_FP128Ty() const { return getTypeID() == PPC_FP128TyID; }
160
161 /// Return true if this is one of the six floating-point types
162 bool isFloatingPointTy() const {
163 return getTypeID() == HalfTyID || getTypeID() == FloatTyID ||
164 getTypeID() == DoubleTyID ||
165 getTypeID() == X86_FP80TyID || getTypeID() == FP128TyID ||
166 getTypeID() == PPC_FP128TyID;
167 }
168
169 const fltSemantics &getFltSemantics() const {
170 switch (getTypeID()) {
171 case HalfTyID: return APFloat::IEEEhalf();
172 case FloatTyID: return APFloat::IEEEsingle();
173 case DoubleTyID: return APFloat::IEEEdouble();
174 case X86_FP80TyID: return APFloat::x87DoubleExtended();
175 case FP128TyID: return APFloat::IEEEquad();
176 case PPC_FP128TyID: return APFloat::PPCDoubleDouble();
177 default: llvm_unreachable("Invalid floating type")::llvm::llvm_unreachable_internal("Invalid floating type", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 177)
;
178 }
179 }
180
181 /// Return true if this is X86 MMX.
182 bool isX86_MMXTy() const { return getTypeID() == X86_MMXTyID; }
183
184 /// Return true if this is a FP type or a vector of FP.
185 bool isFPOrFPVectorTy() const { return getScalarType()->isFloatingPointTy(); }
186
187 /// Return true if this is 'label'.
188 bool isLabelTy() const { return getTypeID() == LabelTyID; }
189
190 /// Return true if this is 'metadata'.
191 bool isMetadataTy() const { return getTypeID() == MetadataTyID; }
192
193 /// Return true if this is 'token'.
194 bool isTokenTy() const { return getTypeID() == TokenTyID; }
195
196 /// True if this is an instance of IntegerType.
197 bool isIntegerTy() const { return getTypeID() == IntegerTyID; }
198
199 /// Return true if this is an IntegerType of the given width.
200 bool isIntegerTy(unsigned Bitwidth) const;
201
202 /// Return true if this is an integer type or a vector of integer types.
203 bool isIntOrIntVectorTy() const { return getScalarType()->isIntegerTy(); }
204
205 /// Return true if this is an integer type or a vector of integer types of
206 /// the given width.
207 bool isIntOrIntVectorTy(unsigned BitWidth) const {
208 return getScalarType()->isIntegerTy(BitWidth);
209 }
210
211 /// Return true if this is an integer type or a pointer type.
212 bool isIntOrPtrTy() const { return isIntegerTy() || isPointerTy(); }
213
214 /// True if this is an instance of FunctionType.
215 bool isFunctionTy() const { return getTypeID() == FunctionTyID; }
216
217 /// True if this is an instance of StructType.
218 bool isStructTy() const { return getTypeID() == StructTyID; }
219
220 /// True if this is an instance of ArrayType.
221 bool isArrayTy() const { return getTypeID() == ArrayTyID; }
222
223 /// True if this is an instance of PointerType.
224 bool isPointerTy() const { return getTypeID() == PointerTyID; }
225
226 /// Return true if this is a pointer type or a vector of pointer types.
227 bool isPtrOrPtrVectorTy() const { return getScalarType()->isPointerTy(); }
228
229 /// True if this is an instance of VectorType.
230 bool isVectorTy() const { return getTypeID() == VectorTyID; }
231
232 /// Return true if this type could be converted with a lossless BitCast to
233 /// type 'Ty'. For example, i8* to i32*. BitCasts are valid for types of the
234 /// same size only where no re-interpretation of the bits is done.
235 /// Determine if this type could be losslessly bitcast to Ty
236 bool canLosslesslyBitCastTo(Type *Ty) const;
237
238 /// Return true if this type is empty, that is, it has no elements or all of
239 /// its elements are empty.
240 bool isEmptyTy() const;
241
242 /// Return true if the type is "first class", meaning it is a valid type for a
243 /// Value.
244 bool isFirstClassType() const {
245 return getTypeID() != FunctionTyID && getTypeID() != VoidTyID;
246 }
247
248 /// Return true if the type is a valid type for a register in codegen. This
249 /// includes all first-class types except struct and array types.
250 bool isSingleValueType() const {
251 return isFloatingPointTy() || isX86_MMXTy() || isIntegerTy() ||
252 isPointerTy() || isVectorTy();
253 }
254
255 /// Return true if the type is an aggregate type. This means it is valid as
256 /// the first operand of an insertvalue or extractvalue instruction. This
257 /// includes struct and array types, but does not include vector types.
258 bool isAggregateType() const {
259 return getTypeID() == StructTyID || getTypeID() == ArrayTyID;
2
Assuming the condition is false
3
Assuming the condition is false
4
Returning zero, which participates in a condition later
260 }
261
262 /// Return true if it makes sense to take the size of this type. To get the
263 /// actual size for a particular target, it is reasonable to use the
264 /// DataLayout subsystem to do this.
265 bool isSized(SmallPtrSetImpl<Type*> *Visited = nullptr) const {
266 // If it's a primitive, it is always sized.
267 if (getTypeID() == IntegerTyID || isFloatingPointTy() ||
268 getTypeID() == PointerTyID ||
269 getTypeID() == X86_MMXTyID)
270 return true;
271 // If it is not something that can have a size (e.g. a function or label),
272 // it doesn't have a size.
273 if (getTypeID() != StructTyID && getTypeID() != ArrayTyID &&
274 getTypeID() != VectorTyID)
275 return false;
276 // Otherwise we have to try harder to decide.
277 return isSizedDerivedType(Visited);
278 }
279
280 /// Return the basic size of this type if it is a primitive type. These are
281 /// fixed by LLVM and are not target-dependent.
282 /// This will return zero if the type does not have a size or is not a
283 /// primitive type.
284 ///
285 /// If this is a scalable vector type, the scalable property will be set and
286 /// the runtime size will be a positive integer multiple of the base size.
287 ///
288 /// Note that this may not reflect the size of memory allocated for an
289 /// instance of the type or the number of bytes that are written when an
290 /// instance of the type is stored to memory. The DataLayout class provides
291 /// additional query functions to provide this information.
292 ///
293 TypeSize getPrimitiveSizeInBits() const LLVM_READONLY__attribute__((__pure__));
294
295 /// If this is a vector type, return the getPrimitiveSizeInBits value for the
296 /// element type. Otherwise return the getPrimitiveSizeInBits value for this
297 /// type.
298 unsigned getScalarSizeInBits() const LLVM_READONLY__attribute__((__pure__));
299
300 /// Return the width of the mantissa of this type. This is only valid on
301 /// floating-point types. If the FP type does not have a stable mantissa (e.g.
302 /// ppc long double), this method returns -1.
303 int getFPMantissaWidth() const;
304
305 /// If this is a vector type, return the element type, otherwise return
306 /// 'this'.
307 Type *getScalarType() const {
308 if (isVectorTy())
309 return getVectorElementType();
310 return const_cast<Type*>(this);
311 }
312
313 //===--------------------------------------------------------------------===//
314 // Type Iteration support.
315 //
316 using subtype_iterator = Type * const *;
317
318 subtype_iterator subtype_begin() const { return ContainedTys; }
319 subtype_iterator subtype_end() const { return &ContainedTys[NumContainedTys];}
320 ArrayRef<Type*> subtypes() const {
321 return makeArrayRef(subtype_begin(), subtype_end());
322 }
323
324 using subtype_reverse_iterator = std::reverse_iterator<subtype_iterator>;
325
326 subtype_reverse_iterator subtype_rbegin() const {
327 return subtype_reverse_iterator(subtype_end());
328 }
329 subtype_reverse_iterator subtype_rend() const {
330 return subtype_reverse_iterator(subtype_begin());
331 }
332
333 /// This method is used to implement the type iterator (defined at the end of
334 /// the file). For derived types, this returns the types 'contained' in the
335 /// derived type.
336 Type *getContainedType(unsigned i) const {
337 assert(i < NumContainedTys && "Index out of range!")((i < NumContainedTys && "Index out of range!") ? static_cast
<void> (0) : __assert_fail ("i < NumContainedTys && \"Index out of range!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 337, __PRETTY_FUNCTION__))
;
338 return ContainedTys[i];
339 }
340
341 /// Return the number of types in the derived type.
342 unsigned getNumContainedTypes() const { return NumContainedTys; }
343
344 //===--------------------------------------------------------------------===//
345 // Helper methods corresponding to subclass methods. This forces a cast to
346 // the specified subclass and calls its accessor. "getVectorNumElements" (for
347 // example) is shorthand for cast<VectorType>(Ty)->getNumElements(). This is
348 // only intended to cover the core methods that are frequently used, helper
349 // methods should not be added here.
350
351 inline unsigned getIntegerBitWidth() const;
352
353 inline Type *getFunctionParamType(unsigned i) const;
354 inline unsigned getFunctionNumParams() const;
355 inline bool isFunctionVarArg() const;
356
357 inline StringRef getStructName() const;
358 inline unsigned getStructNumElements() const;
359 inline Type *getStructElementType(unsigned N) const;
360
361 inline Type *getSequentialElementType() const {
362 assert(isSequentialType(getTypeID()) && "Not a sequential type!")((isSequentialType(getTypeID()) && "Not a sequential type!"
) ? static_cast<void> (0) : __assert_fail ("isSequentialType(getTypeID()) && \"Not a sequential type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 362, __PRETTY_FUNCTION__))
;
363 return ContainedTys[0];
364 }
365
366 inline uint64_t getArrayNumElements() const;
367
368 Type *getArrayElementType() const {
369 assert(getTypeID() == ArrayTyID)((getTypeID() == ArrayTyID) ? static_cast<void> (0) : __assert_fail
("getTypeID() == ArrayTyID", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 369, __PRETTY_FUNCTION__))
;
370 return ContainedTys[0];
371 }
372
373 inline bool getVectorIsScalable() const;
374 inline unsigned getVectorNumElements() const;
375 inline ElementCount getVectorElementCount() const;
376 Type *getVectorElementType() const {
377 assert(getTypeID() == VectorTyID)((getTypeID() == VectorTyID) ? static_cast<void> (0) : __assert_fail
("getTypeID() == VectorTyID", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 377, __PRETTY_FUNCTION__))
;
378 return ContainedTys[0];
379 }
380
381 Type *getPointerElementType() const {
382 assert(getTypeID() == PointerTyID)((getTypeID() == PointerTyID) ? static_cast<void> (0) :
__assert_fail ("getTypeID() == PointerTyID", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 382, __PRETTY_FUNCTION__))
;
383 return ContainedTys[0];
384 }
385
386 /// Given an integer or vector type, change the lane bitwidth to NewBitwidth,
387 /// whilst keeping the old number of lanes.
388 inline Type *getWithNewBitWidth(unsigned NewBitWidth) const;
389
390 /// Given scalar/vector integer type, returns a type with elements twice as
391 /// wide as in the original type. For vectors, preserves element count.
392 inline Type *getExtendedType() const;
393
394 /// Get the address space of this pointer or pointer vector type.
395 inline unsigned getPointerAddressSpace() const;
396
397 //===--------------------------------------------------------------------===//
398 // Static members exported by the Type class itself. Useful for getting
399 // instances of Type.
400 //
401
402 /// Return a type based on an identifier.
403 static Type *getPrimitiveType(LLVMContext &C, TypeID IDNumber);
404
405 //===--------------------------------------------------------------------===//
406 // These are the builtin types that are always available.
407 //
408 static Type *getVoidTy(LLVMContext &C);
409 static Type *getLabelTy(LLVMContext &C);
410 static Type *getHalfTy(LLVMContext &C);
411 static Type *getFloatTy(LLVMContext &C);
412 static Type *getDoubleTy(LLVMContext &C);
413 static Type *getMetadataTy(LLVMContext &C);
414 static Type *getX86_FP80Ty(LLVMContext &C);
415 static Type *getFP128Ty(LLVMContext &C);
416 static Type *getPPC_FP128Ty(LLVMContext &C);
417 static Type *getX86_MMXTy(LLVMContext &C);
418 static Type *getTokenTy(LLVMContext &C);
419 static IntegerType *getIntNTy(LLVMContext &C, unsigned N);
420 static IntegerType *getInt1Ty(LLVMContext &C);
421 static IntegerType *getInt8Ty(LLVMContext &C);
422 static IntegerType *getInt16Ty(LLVMContext &C);
423 static IntegerType *getInt32Ty(LLVMContext &C);
424 static IntegerType *getInt64Ty(LLVMContext &C);
425 static IntegerType *getInt128Ty(LLVMContext &C);
426 template <typename ScalarTy> static Type *getScalarTy(LLVMContext &C) {
427 int noOfBits = sizeof(ScalarTy) * CHAR_BIT8;
428 if (std::is_integral<ScalarTy>::value) {
429 return (Type*) Type::getIntNTy(C, noOfBits);
430 } else if (std::is_floating_point<ScalarTy>::value) {
431 switch (noOfBits) {
432 case 32:
433 return Type::getFloatTy(C);
434 case 64:
435 return Type::getDoubleTy(C);
436 }
437 }
438 llvm_unreachable("Unsupported type in Type::getScalarTy")::llvm::llvm_unreachable_internal("Unsupported type in Type::getScalarTy"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/Type.h"
, 438)
;
439 }
440
441 //===--------------------------------------------------------------------===//
442 // Convenience methods for getting pointer types with one of the above builtin
443 // types as pointee.
444 //
445 static PointerType *getHalfPtrTy(LLVMContext &C, unsigned AS = 0);
446 static PointerType *getFloatPtrTy(LLVMContext &C, unsigned AS = 0);
447 static PointerType *getDoublePtrTy(LLVMContext &C, unsigned AS = 0);
448 static PointerType *getX86_FP80PtrTy(LLVMContext &C, unsigned AS = 0);
449 static PointerType *getFP128PtrTy(LLVMContext &C, unsigned AS = 0);
450 static PointerType *getPPC_FP128PtrTy(LLVMContext &C, unsigned AS = 0);
451 static PointerType *getX86_MMXPtrTy(LLVMContext &C, unsigned AS = 0);
452 static PointerType *getIntNPtrTy(LLVMContext &C, unsigned N, unsigned AS = 0);
453 static PointerType *getInt1PtrTy(LLVMContext &C, unsigned AS = 0);
454 static PointerType *getInt8PtrTy(LLVMContext &C, unsigned AS = 0);
455 static PointerType *getInt16PtrTy(LLVMContext &C, unsigned AS = 0);
456 static PointerType *getInt32PtrTy(LLVMContext &C, unsigned AS = 0);
457 static PointerType *getInt64PtrTy(LLVMContext &C, unsigned AS = 0);
458
459 /// Return a pointer to the current type. This is equivalent to
460 /// PointerType::get(Foo, AddrSpace).
461 PointerType *getPointerTo(unsigned AddrSpace = 0) const;
462
463private:
464 /// Derived types like structures and arrays are sized iff all of the members
465 /// of the type are sized as well. Since asking for their size is relatively
466 /// uncommon, move this operation out-of-line.
467 bool isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited = nullptr) const;
468};
469
470// Printing of types.
471inline raw_ostream &operator<<(raw_ostream &OS, const Type &T) {
472 T.print(OS);
473 return OS;
474}
475
476// allow isa<PointerType>(x) to work without DerivedTypes.h included.
477template <> struct isa_impl<PointerType, Type> {
478 static inline bool doit(const Type &Ty) {
479 return Ty.getTypeID() == Type::PointerTyID;
480 }
481};
482
483// Create wrappers for C Binding types (see CBindingWrapping.h).
484DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)inline Type *unwrap(LLVMTypeRef P) { return reinterpret_cast<
Type*>(P); } inline LLVMTypeRef wrap(const Type *P) { return
reinterpret_cast<LLVMTypeRef>(const_cast<Type*>(
P)); } template<typename T> inline T *unwrap(LLVMTypeRef
P) { return cast<T>(unwrap(P)); }
485
486/* Specialized opaque type conversions.
487 */
488inline Type **unwrap(LLVMTypeRef* Tys) {
489 return reinterpret_cast<Type**>(Tys);
490}
491
492inline LLVMTypeRef *wrap(Type **Tys) {
493 return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
494}
495
496} // end namespace llvm
497
498#endif // LLVM_IR_TYPE_H

/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h

1//===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the IRBuilder class, which is used as a convenient way
10// to create LLVM instructions with a consistent and simplified interface.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_IR_IRBUILDER_H
15#define LLVM_IR_IRBUILDER_H
16
17#include "llvm-c/Types.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/None.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/ADT/Twine.h"
22#include "llvm/IR/BasicBlock.h"
23#include "llvm/IR/Constant.h"
24#include "llvm/IR/ConstantFolder.h"
25#include "llvm/IR/Constants.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/DebugLoc.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/LLVMContext.h"
36#include "llvm/IR/Module.h"
37#include "llvm/IR/Operator.h"
38#include "llvm/IR/Type.h"
39#include "llvm/IR/Value.h"
40#include "llvm/IR/ValueHandle.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/CBindingWrapping.h"
43#include "llvm/Support/Casting.h"
44#include <cassert>
45#include <cstddef>
46#include <cstdint>
47#include <functional>
48#include <utility>
49
50namespace llvm {
51
52class APInt;
53class MDNode;
54class Use;
55
56/// This provides the default implementation of the IRBuilder
57/// 'InsertHelper' method that is called whenever an instruction is created by
58/// IRBuilder and needs to be inserted.
59///
60/// By default, this inserts the instruction at the insertion point.
61class IRBuilderDefaultInserter {
62protected:
63 void InsertHelper(Instruction *I, const Twine &Name,
64 BasicBlock *BB, BasicBlock::iterator InsertPt) const {
65 if (BB) BB->getInstList().insert(InsertPt, I);
66 I->setName(Name);
67 }
68};
69
70/// Provides an 'InsertHelper' that calls a user-provided callback after
71/// performing the default insertion.
72class IRBuilderCallbackInserter : IRBuilderDefaultInserter {
73 std::function<void(Instruction *)> Callback;
74
75public:
76 IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback)
77 : Callback(std::move(Callback)) {}
78
79protected:
80 void InsertHelper(Instruction *I, const Twine &Name,
81 BasicBlock *BB, BasicBlock::iterator InsertPt) const {
82 IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
83 Callback(I);
84 }
85};
86
87/// Common base class shared among various IRBuilders.
88class IRBuilderBase {
89 DebugLoc CurDbgLocation;
90
91protected:
92 BasicBlock *BB;
93 BasicBlock::iterator InsertPt;
94 LLVMContext &Context;
95
96 MDNode *DefaultFPMathTag;
97 FastMathFlags FMF;
98
99 bool IsFPConstrained;
100 fp::ExceptionBehavior DefaultConstrainedExcept;
101 fp::RoundingMode DefaultConstrainedRounding;
102
103 ArrayRef<OperandBundleDef> DefaultOperandBundles;
104
105public:
106 IRBuilderBase(LLVMContext &context, MDNode *FPMathTag = nullptr,
107 ArrayRef<OperandBundleDef> OpBundles = None)
108 : Context(context), DefaultFPMathTag(FPMathTag), IsFPConstrained(false),
109 DefaultConstrainedExcept(fp::ebStrict),
110 DefaultConstrainedRounding(fp::rmDynamic),
111 DefaultOperandBundles(OpBundles) {
112 ClearInsertionPoint();
113 }
114
115 //===--------------------------------------------------------------------===//
116 // Builder configuration methods
117 //===--------------------------------------------------------------------===//
118
119 /// Clear the insertion point: created instructions will not be
120 /// inserted into a block.
121 void ClearInsertionPoint() {
122 BB = nullptr;
123 InsertPt = BasicBlock::iterator();
124 }
125
126 BasicBlock *GetInsertBlock() const { return BB; }
127 BasicBlock::iterator GetInsertPoint() const { return InsertPt; }
128 LLVMContext &getContext() const { return Context; }
129
130 /// This specifies that created instructions should be appended to the
131 /// end of the specified block.
132 void SetInsertPoint(BasicBlock *TheBB) {
133 BB = TheBB;
134 InsertPt = BB->end();
135 }
136
137 /// This specifies that created instructions should be inserted before
138 /// the specified instruction.
139 void SetInsertPoint(Instruction *I) {
140 BB = I->getParent();
141 InsertPt = I->getIterator();
142 assert(InsertPt != BB->end() && "Can't read debug loc from end()")((InsertPt != BB->end() && "Can't read debug loc from end()"
) ? static_cast<void> (0) : __assert_fail ("InsertPt != BB->end() && \"Can't read debug loc from end()\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h"
, 142, __PRETTY_FUNCTION__))
;
143 SetCurrentDebugLocation(I->getDebugLoc());
144 }
145
146 /// This specifies that created instructions should be inserted at the
147 /// specified point.
148 void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) {
149 BB = TheBB;
150 InsertPt = IP;
151 if (IP != TheBB->end())
152 SetCurrentDebugLocation(IP->getDebugLoc());
153 }
154
155 /// Set location information used by debugging information.
156 void SetCurrentDebugLocation(DebugLoc L) { CurDbgLocation = std::move(L); }
157
158 /// Get location information used by debugging information.
159 const DebugLoc &getCurrentDebugLocation() const { return CurDbgLocation; }
160
161 /// If this builder has a current debug location, set it on the
162 /// specified instruction.
163 void SetInstDebugLocation(Instruction *I) const {
164 if (CurDbgLocation)
165 I->setDebugLoc(CurDbgLocation);
166 }
167
168 /// Get the return type of the current function that we're emitting
169 /// into.
170 Type *getCurrentFunctionReturnType() const;
171
172 /// InsertPoint - A saved insertion point.
173 class InsertPoint {
174 BasicBlock *Block = nullptr;
175 BasicBlock::iterator Point;
176
177 public:
178 /// Creates a new insertion point which doesn't point to anything.
179 InsertPoint() = default;
180
181 /// Creates a new insertion point at the given location.
182 InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint)
183 : Block(InsertBlock), Point(InsertPoint) {}
184
185 /// Returns true if this insert point is set.
186 bool isSet() const { return (Block != nullptr); }
187
188 BasicBlock *getBlock() const { return Block; }
189 BasicBlock::iterator getPoint() const { return Point; }
190 };
191
192 /// Returns the current insert point.
193 InsertPoint saveIP() const {
194 return InsertPoint(GetInsertBlock(), GetInsertPoint());
195 }
196
197 /// Returns the current insert point, clearing it in the process.
198 InsertPoint saveAndClearIP() {
199 InsertPoint IP(GetInsertBlock(), GetInsertPoint());
200 ClearInsertionPoint();
201 return IP;
202 }
203
204 /// Sets the current insert point to a previously-saved location.
205 void restoreIP(InsertPoint IP) {
206 if (IP.isSet())
207 SetInsertPoint(IP.getBlock(), IP.getPoint());
208 else
209 ClearInsertionPoint();
210 }
211
212 /// Get the floating point math metadata being used.
213 MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; }
214
215 /// Get the flags to be applied to created floating point ops
216 FastMathFlags getFastMathFlags() const { return FMF; }
217
218 /// Clear the fast-math flags.
219 void clearFastMathFlags() { FMF.clear(); }
220
221 /// Set the floating point math metadata to be used.
222 void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; }
223
224 /// Set the fast-math flags to be used with generated fp-math operators
225 void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; }
226
227 /// Enable/Disable use of constrained floating point math. When
228 /// enabled the CreateF<op>() calls instead create constrained
229 /// floating point intrinsic calls. Fast math flags are unaffected
230 /// by this setting.
231 void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; }
232
233 /// Query for the use of constrained floating point math
234 bool getIsFPConstrained() { return IsFPConstrained; }
235
236 /// Set the exception handling to be used with constrained floating point
237 void setDefaultConstrainedExcept(fp::ExceptionBehavior NewExcept) {
238 DefaultConstrainedExcept = NewExcept;
239 }
240
241 /// Set the rounding mode handling to be used with constrained floating point
242 void setDefaultConstrainedRounding(fp::RoundingMode NewRounding) {
243 DefaultConstrainedRounding = NewRounding;
244 }
245
246 /// Get the exception handling used with constrained floating point
247 fp::ExceptionBehavior getDefaultConstrainedExcept() {
248 return DefaultConstrainedExcept;
249 }
250
251 /// Get the rounding mode handling used with constrained floating point
252 fp::RoundingMode getDefaultConstrainedRounding() {
253 return DefaultConstrainedRounding;
254 }
255
256 void setConstrainedFPFunctionAttr() {
257 assert(BB && "Must have a basic block to set any function attributes!")((BB && "Must have a basic block to set any function attributes!"
) ? static_cast<void> (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include/llvm/IR/IRBuilder.h"
, 257, __PRETTY_FUNCTION__))
;
258
259 Function *F = BB->getParent();
260 if (!F->hasFnAttribute(Attribute::StrictFP)) {
261 F->addFnAttr(Attribute::StrictFP);
262 }
263 }
264
265 void setConstrainedFPCallAttr(CallInst *I) {
266 if (!I->hasFnAttr(Attribute::StrictFP))
267 I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
268 }
269
270 //===--------------------------------------------------------------------===//
271 // RAII helpers.
272 //===--------------------------------------------------------------------===//
273
274 // RAII object that stores the current insertion point and restores it
275 // when the object is destroyed. This includes the debug location.
276 class InsertPointGuard {
277 IRBuilderBase &Builder;
278 AssertingVH<BasicBlock> Block;
279 BasicBlock::iterator Point;
280 DebugLoc DbgLoc;
281
282 public:
283 InsertPointGuard(IRBuilderBase &B)
284 : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()),
285 DbgLoc(B.getCurrentDebugLocation()) {}
286
287 InsertPointGuard(const InsertPointGuard &) = delete;
288 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
289
290 ~InsertPointGuard() {
291 Builder.restoreIP(InsertPoint(Block, Point));
292 Builder.SetCurrentDebugLocation(DbgLoc);
293 }
294 };
295
296 // RAII object that stores the current fast math settings and restores
297 // them when the object is destroyed.
298 class FastMathFlagGuard {
299 IRBuilderBase &Builder;
300 FastMathFlags FMF;
301 MDNode *FPMathTag;
302
303 public:
304 FastMathFlagGuard(IRBuilderBase &B)
305 : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag) {}
306
307 FastMathFlagGuard(const FastMathFlagGuard &) = delete;
308 FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete;
309
310 ~FastMathFlagGuard() {
311 Builder.FMF = FMF;
312 Builder.DefaultFPMathTag = FPMathTag;
313 }
314 };
315
316 //===--------------------------------------------------------------------===//
317 // Miscellaneous creation methods.
318 //===--------------------------------------------------------------------===//
319
320 /// Make a new global variable with initializer type i8*
321 ///
322 /// Make a new global variable with an initializer that has array of i8 type
323 /// filled in with the null terminated string value specified. The new global
324 /// variable will be marked mergable with any others of the same contents. If
325 /// Name is specified, it is the name of the global variable created.
326 GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "",
327 unsigned AddressSpace = 0);
328
329 /// Get a constant value representing either true or false.
330 ConstantInt *getInt1(bool V) {
331 return ConstantInt::get(getInt1Ty(), V);
332 }
333
334 /// Get the constant value for i1 true.
335 ConstantInt *getTrue() {
336 return ConstantInt::getTrue(Context);
337 }
338
339 /// Get the constant value for i1 false.
340 ConstantInt *getFalse() {
341 return ConstantInt::getFalse(Context);
342 }
343
344 /// Get a constant 8-bit value.
345 ConstantInt *getInt8(uint8_t C) {
346 return ConstantInt::get(getInt8Ty(), C);
347 }
348
349 /// Get a constant 16-bit value.
350 ConstantInt *getInt16(uint16_t C) {
351 return ConstantInt::get(getInt16Ty(), C);
352 }
353
354 /// Get a constant 32-bit value.
355 ConstantInt *getInt32(uint32_t C) {
356 return ConstantInt::get(getInt32Ty(), C);
357 }
358
359 /// Get a constant 64-bit value.
360 ConstantInt *getInt64(uint64_t C) {
361 return ConstantInt::get(getInt64Ty(), C);
362 }
363
364 /// Get a constant N-bit value, zero extended or truncated from
365 /// a 64-bit value.
366 ConstantInt *getIntN(unsigned N, uint64_t C) {
367 return ConstantInt::get(getIntNTy(N), C);
368 }
369
370 /// Get a constant integer value.
371 ConstantInt *getInt(const APInt &AI) {
372 return ConstantInt::get(Context, AI);
373 }
374
375 //===--------------------------------------------------------------------===//
376 // Type creation methods
377 //===--------------------------------------------------------------------===//
378
379 /// Fetch the type representing a single bit
380 IntegerType *getInt1Ty() {
381 return Type::getInt1Ty(Context);
382 }
383
384 /// Fetch the type representing an 8-bit integer.
385 IntegerType *getInt8Ty() {
386 return Type::getInt8Ty(Context);
387 }
388
389 /// Fetch the type representing a 16-bit integer.
390 IntegerType *getInt16Ty() {
391 return Type::getInt16Ty(Context);
392 }
393
394 /// Fetch the type representing a 32-bit integer.
395 IntegerType *getInt32Ty() {
396 return Type::getInt32Ty(Context);
397 }
398
399 /// Fetch the type representing a 64-bit integer.
400 IntegerType *getInt64Ty() {
401 return Type::getInt64Ty(Context);
402 }
403
404 /// Fetch the type representing a 128-bit integer.
405 IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); }
406
407 /// Fetch the type representing an N-bit integer.
408 IntegerType *getIntNTy(unsigned N) {
409 return Type::getIntNTy(Context, N);
410 }
411
412 /// Fetch the type representing a 16-bit floating point value.
413 Type *getHalfTy() {
414 return Type::getHalfTy(Context);
415 }
416
417 /// Fetch the type representing a 32-bit floating point value.
418 Type *getFloatTy() {
419 return Type::getFloatTy(Context);
420 }
421
422 /// Fetch the type representing a 64-bit floating point value.
423 Type *getDoubleTy() {
424 return Type::getDoubleTy(Context);
425 }
426
427 /// Fetch the type representing void.
428 Type *getVoidTy() {
429 return Type::getVoidTy(Context);
430 }
431
432 /// Fetch the type representing a pointer to an 8-bit integer value.
433 PointerType *getInt8PtrTy(unsigned AddrSpace = 0) {
434 return Type::getInt8PtrTy(Context, AddrSpace);
435 }
436
437 /// Fetch the type representing a pointer to an integer value.
438 IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) {
439 return DL.getIntPtrType(Context, AddrSpace);
440 }
441
442 //===--------------------------------------------------------------------===//
443 // Intrinsic creation methods
444 //===--------------------------------------------------------------------===//
445
446 /// Create and insert a memset to the specified pointer and the
447 /// specified value.
448 ///
449 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
450 /// specified, it will be added to the instruction. Likewise with alias.scope
451 /// and noalias tags.
452 CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size,
453 MaybeAlign Align, bool isVolatile = false,
454 MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr,
455 MDNode *NoAliasTag = nullptr) {
456 return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile,
457 TBAATag, ScopeTag, NoAliasTag);
458 }
459
460 CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, MaybeAlign Align,
461 bool isVolatile = false, MDNode *TBAATag = nullptr,
462 MDNode *ScopeTag = nullptr,
463 MDNode *NoAliasTag = nullptr);
464
465 /// Create and insert an element unordered-atomic memset of the region of
466 /// memory starting at the given pointer to the given value.
467 ///
468 /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is
469 /// specified, it will be added to the instruction. Likewise with alias.scope
470 /// and noalias tags.
471 /// FIXME: Remove this function once transition to Align is over.
472 /// Use the version that takes Align instead of this one.
473 LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
474 CallInst *CreateElementUnorderedAtomicMemSet(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
475 Value *Ptr, Value *Val, uint64_t Size, unsigned Alignment,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
476 uint32_t ElementSize, MDNode *TBAATag = nullptr,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
477 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
478 "Use the version that takes Align instead of this one")CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, uint64_t Size, unsigned Alignment, uint32_t ElementSize
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
{
479 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
480 Align(Alignment), ElementSize,
481 TBAATag, ScopeTag, NoAliasTag);
482 }
483
484 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
485 uint64_t Size, Align Alignment,
486 uint32_t ElementSize,
487 MDNode *TBAATag = nullptr,
488 MDNode *ScopeTag = nullptr,
489 MDNode *NoAliasTag = nullptr) {
490 return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size),
491 Align(Alignment), ElementSize,
492 TBAATag, ScopeTag, NoAliasTag);
493 }
494
495 /// FIXME: Remove this function once transition to Align is over.
496 /// Use the version that takes Align instead of this one.
497 LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, Value *Size, unsigned Alignment, uint32_t ElementSize,
MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
498 CallInst *CreateElementUnorderedAtomicMemSet(CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, Value *Size, unsigned Alignment, uint32_t ElementSize,
MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
499 Value *Ptr, Value *Val, Value *Size, unsigned Alignment,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, Value *Size, unsigned Alignment, uint32_t ElementSize,
MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
500 uint32_t ElementSize, MDNode *TBAATag = nullptr,CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, Value *Size, unsigned Alignment, uint32_t ElementSize,
MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
501 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, Value *Size, unsigned Alignment, uint32_t ElementSize,
MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
502 "Use the version that takes Align instead of this one")CallInst *CreateElementUnorderedAtomicMemSet( Value *Ptr, Value
*Val, Value *Size, unsigned Alignment, uint32_t ElementSize,
MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes Align instead of this one"
)))
{
503 return CreateElementUnorderedAtomicMemSet(Ptr, Val, Size, Align(Alignment),
504 ElementSize, TBAATag, ScopeTag,
505 NoAliasTag);
506 }
507
508 CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val,
509 Value *Size, Align Alignment,
510 uint32_t ElementSize,
511 MDNode *TBAATag = nullptr,
512 MDNode *ScopeTag = nullptr,
513 MDNode *NoAliasTag = nullptr);
514
515 /// Create and insert a memcpy between the specified pointers.
516 ///
517 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
518 /// specified, it will be added to the instruction. Likewise with alias.scope
519 /// and noalias tags.
520 /// FIXME: Remove this function once transition to Align is over.
521 /// Use the version that takes MaybeAlign instead of this one.
522 LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
523 CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
524 unsigned SrcAlign, uint64_t Size,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
525 bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
526 MDNode *TBAAStructTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
527 MDNode *ScopeTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
528 MDNode *NoAliasTag = nullptr),CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
529 "Use the version that takes MaybeAlign instead")CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr,
MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
{
530 return CreateMemCpy(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
531 getInt64(Size), isVolatile, TBAATag, TBAAStructTag,
532 ScopeTag, NoAliasTag);
533 }
534
535 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
536 MaybeAlign SrcAlign, uint64_t Size,
537 bool isVolatile = false, MDNode *TBAATag = nullptr,
538 MDNode *TBAAStructTag = nullptr,
539 MDNode *ScopeTag = nullptr,
540 MDNode *NoAliasTag = nullptr) {
541 return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
542 isVolatile, TBAATag, TBAAStructTag, ScopeTag,
543 NoAliasTag);
544 }
545
546 /// FIXME: Remove this function once transition to Align is over.
547 /// Use the version that takes MaybeAlign instead of this one.
548 LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
549 CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
550 unsigned SrcAlign, Value *Size,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
551 bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
552 MDNode *TBAAStructTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
553 MDNode *ScopeTag = nullptr,CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
554 MDNode *NoAliasTag = nullptr),CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
555 "Use the version that takes MaybeAlign instead")CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *
Src, unsigned SrcAlign, Value *Size, bool isVolatile = false,
MDNode *TBAATag = nullptr, MDNode *TBAAStructTag = nullptr, MDNode
*ScopeTag = nullptr, MDNode *NoAliasTag = nullptr) __attribute__
((deprecated("Use the version that takes MaybeAlign instead")
))
;
556 CallInst *CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src,
557 MaybeAlign SrcAlign, Value *Size,
558 bool isVolatile = false, MDNode *TBAATag = nullptr,
559 MDNode *TBAAStructTag = nullptr,
560 MDNode *ScopeTag = nullptr,
561 MDNode *NoAliasTag = nullptr);
562
563 /// Create and insert an element unordered-atomic memcpy between the
564 /// specified pointers.
565 ///
566 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively.
567 ///
568 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
569 /// specified, it will be added to the instruction. Likewise with alias.scope
570 /// and noalias tags.
571 CallInst *CreateElementUnorderedAtomicMemCpy(
572 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
573 uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr,
574 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
575 MDNode *NoAliasTag = nullptr) {
576 return CreateElementUnorderedAtomicMemCpy(
577 Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag,
578 TBAAStructTag, ScopeTag, NoAliasTag);
579 }
580
581 CallInst *CreateElementUnorderedAtomicMemCpy(
582 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
583 uint32_t ElementSize, MDNode *TBAATag = nullptr,
584 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
585 MDNode *NoAliasTag = nullptr);
586
587 /// Create and insert a memmove between the specified
588 /// pointers.
589 ///
590 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
591 /// specified, it will be added to the instruction. Likewise with alias.scope
592 /// and noalias tags.
593 /// FIXME: Remove this function once transition to Align is over.
594 /// Use the version that takes MaybeAlign instead of this one.
595 LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
596 CallInst *CreateMemMove(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
597 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
598 uint64_t Size, bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
599 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
600 "Use the version that takes MaybeAlign")CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, uint64_t Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
{
601 return CreateMemMove(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
602 getInt64(Size), isVolatile, TBAATag, ScopeTag,
603 NoAliasTag);
604 }
605 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
606 MaybeAlign SrcAlign, uint64_t Size,
607 bool isVolatile = false, MDNode *TBAATag = nullptr,
608 MDNode *ScopeTag = nullptr,
609 MDNode *NoAliasTag = nullptr) {
610 return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size),
611 isVolatile, TBAATag, ScopeTag, NoAliasTag);
612 }
613 /// FIXME: Remove this function once transition to Align is over.
614 /// Use the version that takes MaybeAlign instead of this one.
615 LLVM_ATTRIBUTE_DEPRECATED(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, Value *Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
616 CallInst *CreateMemMove(CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, Value *Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
617 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, Value *Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
618 Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr,CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, Value *Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
619 MDNode *ScopeTag = nullptr, MDNode *NoAliasTag = nullptr),CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, Value *Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
620 "Use the version that takes MaybeAlign")CallInst *CreateMemMove( Value *Dst, unsigned DstAlign, Value
*Src, unsigned SrcAlign, Value *Size, bool isVolatile = false
, MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, MDNode
*NoAliasTag = nullptr) __attribute__((deprecated("Use the version that takes MaybeAlign"
)))
{
621 return CreateMemMove(Dst, MaybeAlign(DstAlign), Src, MaybeAlign(SrcAlign),
622 Size, isVolatile, TBAATag, ScopeTag, NoAliasTag);
623 }
624 CallInst *CreateMemMove(Value *Dst, MaybeAlign DstAlign, Value *Src,
625 MaybeAlign SrcAlign, Value *Size,
626 bool isVolatile = false, MDNode *TBAATag = nullptr,
627 MDNode *ScopeTag = nullptr,
628 MDNode *NoAliasTag = nullptr);
629
630 /// \brief Create and insert an element unordered-atomic memmove between the
631 /// specified pointers.
632 ///
633 /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers,
634 /// respectively.
635 ///
636 /// If the pointers aren't i8*, they will be converted. If a TBAA tag is
637 /// specified, it will be added to the instruction. Likewise with alias.scope
638 /// and noalias tags.
639 CallInst *CreateElementUnorderedAtomicMemMove(
640 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
641 uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr,
642 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
643 MDNode *NoAliasTag = nullptr) {
644 return CreateElementUnorderedAtomicMemMove(
645 Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag,
646 TBAAStructTag, ScopeTag, NoAliasTag);
647 }
648
649 CallInst *CreateElementUnorderedAtomicMemMove(
650 Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
651 uint32_t ElementSize, MDNode *TBAATag = nullptr,
652 MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr,
653 MDNode *NoAliasTag = nullptr);
654
655 /// Create a vector fadd reduction intrinsic of the source vector.
656 /// The first parameter is a scalar accumulator value for ordered reductions.
657 CallInst *CreateFAddReduce(Value *Acc, Value *Src);
658
659 /// Create a vector fmul reduction intrinsic of the source vector.
660 /// The first parameter is a scalar accumulator value for ordered reductions.
661 CallInst *CreateFMulReduce(Value *Acc, Value *Src);
662
663 /// Create a vector int add reduction intrinsic of the source vector.
664 CallInst *CreateAddReduce(Value *Src);
665
666 /// Create a vector int mul reduction intrinsic of the source vector.
667 CallInst *CreateMulReduce(Value *Src);
668
669 /// Create a vector int AND reduction intrinsic of the source vector.
670 CallInst *CreateAndReduce(Value *Src);
671
672 /// Create a vector int OR reduction intrinsic of the source vector.
673 CallInst *CreateOrReduce(Value *Src);
674
675 /// Create a vector int XOR reduction intrinsic of the source vector.
676 CallInst *CreateXorReduce(Value *Src);
677
678 /// Create a vector integer max reduction intrinsic of the source
679 /// vector.
680 CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false);
681
682 /// Create a vector integer min reduction intrinsic of the source
683 /// vector.
684 CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false);
685
686 /// Create a vector float max reduction intrinsic of the source
687 /// vector.
688 CallInst *CreateFPMaxReduce(Value *Src, bool NoNaN = false);
689
690 /// Create a vector float min reduction intrinsic of the source
691 /// vector.
692 CallInst *CreateFPMinReduce(Value *Src, bool NoNaN = false);
693
694 /// Create a lifetime.start intrinsic.
695 ///
696 /// If the pointer isn't i8* it will be converted.
697 CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr);
698
699 /// Create a lifetime.end intrinsic.
700 ///
701 /// If the pointer isn't i8* it will be converted.
702 CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
703
704 /// Create a call to invariant.start intrinsic.
705 ///
706 /// If the pointer isn't i8* it will be converted.
707 CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr);
708
709 /// Create a call to Masked Load intrinsic
710 CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask,
711 Value *PassThru = nullptr, const Twine &Name = "");
712
713 /// Create a call to Masked Store intrinsic
714 CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align,
715 Value *Mask);
716
717 /// Create a call to Masked Gather intrinsic
718 CallInst *CreateMaskedGather(Value *Ptrs, unsigned Align,
719 Value *Mask = nullptr,
720 Value *PassThru = nullptr,
721 const Twine& Name = "");
722
723 /// Create a call to Masked Scatter intrinsic
724 CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, unsigned Align,
725 Value *Mask = nullptr);
726
727 /// Create an assume intrinsic call that allows the optimizer to
728 /// assume that the provided condition will be true.
729 CallInst *CreateAssumption(Value *Cond);
730
731 /// Create a call to the experimental.gc.statepoint intrinsic to
732 /// start a new statepoint sequence.
733 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
734 Value *ActualCallee,
735 ArrayRef<Value *> CallArgs,
736 ArrayRef<Value *> DeoptArgs,
737 ArrayRef<Value *> GCArgs,
738 const Twine &Name = "");
739
740 /// Create a call to the experimental.gc.statepoint intrinsic to
741 /// start a new statepoint sequence.
742 CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes,
743 Value *ActualCallee, uint32_t Flags,
744 ArrayRef<Use> CallArgs,
745