Bug Summary

File:build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
Warning:line 1741, column 7
Forming reference to null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name MemorySanitizer.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Transforms/Instrumentation -I include -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-09-04-125545-48738-1 -x c++ /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
1//===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of MemorySanitizer, a detector of uninitialized
11/// reads.
12///
13/// The algorithm of the tool is similar to Memcheck
14/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15/// byte of the application memory, poison the shadow of the malloc-ed
16/// or alloca-ed memory, load the shadow bits on every memory read,
17/// propagate the shadow bits through some of the arithmetic
18/// instruction (including MOV), store the shadow bits on every memory
19/// write, report a bug on some other instructions (e.g. JMP) if the
20/// associated shadow is poisoned.
21///
22/// But there are differences too. The first and the major one:
23/// compiler instrumentation instead of binary instrumentation. This
24/// gives us much better register allocation, possible compiler
25/// optimizations and a fast start-up. But this brings the major issue
26/// as well: msan needs to see all program events, including system
27/// calls and reads/writes in system libraries, so we either need to
28/// compile *everything* with msan or use a binary translation
29/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30/// Another difference from Memcheck is that we use 8 shadow bits per
31/// byte of application memory and use a direct shadow mapping. This
32/// greatly simplifies the instrumentation code and avoids races on
33/// shadow updates (Memcheck is single-threaded so races are not a
34/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35/// path storage that uses 8 bits per byte).
36///
37/// The default value of shadow is 0, which means "clean" (not poisoned).
38///
39/// Every module initializer should call __msan_init to ensure that the
40/// shadow memory is ready. On error, __msan_warning is called. Since
41/// parameters and return values may be passed via registers, we have a
42/// specialized thread-local shadow for return values
43/// (__msan_retval_tls) and parameters (__msan_param_tls).
44///
45/// Origin tracking.
46///
47/// MemorySanitizer can track origins (allocation points) of all uninitialized
48/// values. This behavior is controlled with a flag (msan-track-origins) and is
49/// disabled by default.
50///
51/// Origins are 4-byte values created and interpreted by the runtime library.
52/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53/// of application memory. Propagation of origins is basically a bunch of
54/// "select" instructions that pick the origin of a dirty argument, if an
55/// instruction has one.
56///
57/// Every 4 aligned, consecutive bytes of application memory have one origin
58/// value associated with them. If these bytes contain uninitialized data
59/// coming from 2 different allocations, the last store wins. Because of this,
60/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61/// practice.
62///
63/// Origins are meaningless for fully initialized values, so MemorySanitizer
64/// avoids storing origin to memory when a fully initialized value is stored.
65/// This way it avoids needless overwriting origin of the 4-byte region on
66/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67///
68/// Atomic handling.
69///
70/// Ideally, every atomic store of application value should update the
71/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72/// of two disjoint locations can not be done without severe slowdown.
73///
74/// Therefore, we implement an approximation that may err on the safe side.
75/// In this implementation, every atomically accessed location in the program
76/// may only change from (partially) uninitialized to fully initialized, but
77/// not the other way around. We load the shadow _after_ the application load,
78/// and we store the shadow _before_ the app store. Also, we always store clean
79/// shadow (if the application store is atomic). This way, if the store-load
80/// pair constitutes a happens-before arc, shadow store and load are correctly
81/// ordered such that the load will get either the value that was stored, or
82/// some later value (which is always clean).
83///
84/// This does not work very well with Compare-And-Swap (CAS) and
85/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86/// must store the new shadow before the app operation, and load the shadow
87/// after the app operation. Computers don't work this way. Current
88/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89/// value. It implements the store part as a simple atomic store by storing a
90/// clean shadow.
91///
92/// Instrumenting inline assembly.
93///
94/// For inline assembly code LLVM has little idea about which memory locations
95/// become initialized depending on the arguments. It can be possible to figure
96/// out which arguments are meant to point to inputs and outputs, but the
97/// actual semantics can be only visible at runtime. In the Linux kernel it's
98/// also possible that the arguments only indicate the offset for a base taken
99/// from a segment register, so it's dangerous to treat any asm() arguments as
100/// pointers. We take a conservative approach generating calls to
101/// __msan_instrument_asm_store(ptr, size)
102/// , which defer the memory unpoisoning to the runtime library.
103/// The latter can perform more complex address checks to figure out whether
104/// it's safe to touch the shadow memory.
105/// Like with atomic operations, we call __msan_instrument_asm_store() before
106/// the assembly call, so that changes to the shadow memory will be seen by
107/// other threads together with main memory initialization.
108///
109/// KernelMemorySanitizer (KMSAN) implementation.
110///
111/// The major differences between KMSAN and MSan instrumentation are:
112/// - KMSAN always tracks the origins and implies msan-keep-going=true;
113/// - KMSAN allocates shadow and origin memory for each page separately, so
114/// there are no explicit accesses to shadow and origin in the
115/// instrumentation.
116/// Shadow and origin values for a particular X-byte memory location
117/// (X=1,2,4,8) are accessed through pointers obtained via the
118/// __msan_metadata_ptr_for_load_X(ptr)
119/// __msan_metadata_ptr_for_store_X(ptr)
120/// functions. The corresponding functions check that the X-byte accesses
121/// are possible and returns the pointers to shadow and origin memory.
122/// Arbitrary sized accesses are handled with:
123/// __msan_metadata_ptr_for_load_n(ptr, size)
124/// __msan_metadata_ptr_for_store_n(ptr, size);
125/// - TLS variables are stored in a single per-task struct. A call to a
126/// function __msan_get_context_state() returning a pointer to that struct
127/// is inserted into every instrumented function before the entry block;
128/// - __msan_warning() takes a 32-bit origin parameter;
129/// - local variables are poisoned with __msan_poison_alloca() upon function
130/// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131/// function;
132/// - the pass doesn't declare any global variables or add global constructors
133/// to the translation unit.
134///
135/// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136/// calls, making sure we're on the safe side wrt. possible false positives.
137///
138/// KernelMemorySanitizer only supports X86_64 at the moment.
139///
140//
141// FIXME: This sanitizer does not yet handle scalable vectors
142//
143//===----------------------------------------------------------------------===//
144
145#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
146#include "llvm/ADT/APInt.h"
147#include "llvm/ADT/ArrayRef.h"
148#include "llvm/ADT/DepthFirstIterator.h"
149#include "llvm/ADT/SetVector.h"
150#include "llvm/ADT/SmallString.h"
151#include "llvm/ADT/SmallVector.h"
152#include "llvm/ADT/StringExtras.h"
153#include "llvm/ADT/StringRef.h"
154#include "llvm/ADT/Triple.h"
155#include "llvm/Analysis/TargetLibraryInfo.h"
156#include "llvm/Analysis/ValueTracking.h"
157#include "llvm/IR/Argument.h"
158#include "llvm/IR/Attributes.h"
159#include "llvm/IR/BasicBlock.h"
160#include "llvm/IR/CallingConv.h"
161#include "llvm/IR/Constant.h"
162#include "llvm/IR/Constants.h"
163#include "llvm/IR/DataLayout.h"
164#include "llvm/IR/DerivedTypes.h"
165#include "llvm/IR/Function.h"
166#include "llvm/IR/GlobalValue.h"
167#include "llvm/IR/GlobalVariable.h"
168#include "llvm/IR/IRBuilder.h"
169#include "llvm/IR/InlineAsm.h"
170#include "llvm/IR/InstVisitor.h"
171#include "llvm/IR/InstrTypes.h"
172#include "llvm/IR/Instruction.h"
173#include "llvm/IR/Instructions.h"
174#include "llvm/IR/IntrinsicInst.h"
175#include "llvm/IR/Intrinsics.h"
176#include "llvm/IR/IntrinsicsX86.h"
177#include "llvm/IR/MDBuilder.h"
178#include "llvm/IR/Module.h"
179#include "llvm/IR/Type.h"
180#include "llvm/IR/Value.h"
181#include "llvm/IR/ValueMap.h"
182#include "llvm/Support/Alignment.h"
183#include "llvm/Support/AtomicOrdering.h"
184#include "llvm/Support/Casting.h"
185#include "llvm/Support/CommandLine.h"
186#include "llvm/Support/Debug.h"
187#include "llvm/Support/ErrorHandling.h"
188#include "llvm/Support/MathExtras.h"
189#include "llvm/Support/raw_ostream.h"
190#include "llvm/Transforms/Utils/BasicBlockUtils.h"
191#include "llvm/Transforms/Utils/Local.h"
192#include "llvm/Transforms/Utils/ModuleUtils.h"
193#include <algorithm>
194#include <cassert>
195#include <cstddef>
196#include <cstdint>
197#include <memory>
198#include <string>
199#include <tuple>
200
201using namespace llvm;
202
203#define DEBUG_TYPE"msan" "msan"
204
205static const unsigned kOriginSize = 4;
206static const Align kMinOriginAlignment = Align(4);
207static const Align kShadowTLSAlignment = Align(8);
208
209// These constants must be kept in sync with the ones in msan.h.
210static const unsigned kParamTLSSize = 800;
211static const unsigned kRetvalTLSSize = 800;
212
213// Accesses sizes are powers of two: 1, 2, 4, 8.
214static const size_t kNumberOfAccessSizes = 4;
215
216/// Track origins of uninitialized values.
217///
218/// Adds a section to MemorySanitizer report that points to the allocation
219/// (stack or heap) the uninitialized bits came from originally.
220static cl::opt<int> ClTrackOrigins(
221 "msan-track-origins",
222 cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden,
223 cl::init(0));
224
225static cl::opt<bool> ClKeepGoing("msan-keep-going",
226 cl::desc("keep going after reporting a UMR"),
227 cl::Hidden, cl::init(false));
228
229static cl::opt<bool>
230 ClPoisonStack("msan-poison-stack",
231 cl::desc("poison uninitialized stack variables"), cl::Hidden,
232 cl::init(true));
233
234static cl::opt<bool> ClPoisonStackWithCall(
235 "msan-poison-stack-with-call",
236 cl::desc("poison uninitialized stack variables with a call"), cl::Hidden,
237 cl::init(false));
238
239static cl::opt<int> ClPoisonStackPattern(
240 "msan-poison-stack-pattern",
241 cl::desc("poison uninitialized stack variables with the given pattern"),
242 cl::Hidden, cl::init(0xff));
243
244static cl::opt<bool>
245 ClPrintStackNames("msan-print-stack-names",
246 cl::desc("Print name of local stack variable"),
247 cl::Hidden, cl::init(true));
248
249static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
250 cl::desc("poison undef temps"), cl::Hidden,
251 cl::init(true));
252
253static cl::opt<bool>
254 ClHandleICmp("msan-handle-icmp",
255 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
256 cl::Hidden, cl::init(true));
257
258static cl::opt<bool>
259 ClHandleICmpExact("msan-handle-icmp-exact",
260 cl::desc("exact handling of relational integer ICmp"),
261 cl::Hidden, cl::init(false));
262
263static cl::opt<bool> ClHandleLifetimeIntrinsics(
264 "msan-handle-lifetime-intrinsics",
265 cl::desc(
266 "when possible, poison scoped variables at the beginning of the scope "
267 "(slower, but more precise)"),
268 cl::Hidden, cl::init(true));
269
270// When compiling the Linux kernel, we sometimes see false positives related to
271// MSan being unable to understand that inline assembly calls may initialize
272// local variables.
273// This flag makes the compiler conservatively unpoison every memory location
274// passed into an assembly call. Note that this may cause false positives.
275// Because it's impossible to figure out the array sizes, we can only unpoison
276// the first sizeof(type) bytes for each type* pointer.
277// The instrumentation is only enabled in KMSAN builds, and only if
278// -msan-handle-asm-conservative is on. This is done because we may want to
279// quickly disable assembly instrumentation when it breaks.
280static cl::opt<bool> ClHandleAsmConservative(
281 "msan-handle-asm-conservative",
282 cl::desc("conservative handling of inline assembly"), cl::Hidden,
283 cl::init(true));
284
285// This flag controls whether we check the shadow of the address
286// operand of load or store. Such bugs are very rare, since load from
287// a garbage address typically results in SEGV, but still happen
288// (e.g. only lower bits of address are garbage, or the access happens
289// early at program startup where malloc-ed memory is more likely to
290// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
291static cl::opt<bool> ClCheckAccessAddress(
292 "msan-check-access-address",
293 cl::desc("report accesses through a pointer which has poisoned shadow"),
294 cl::Hidden, cl::init(true));
295
296static cl::opt<bool> ClEagerChecks(
297 "msan-eager-checks",
298 cl::desc("check arguments and return values at function call boundaries"),
299 cl::Hidden, cl::init(false));
300
301static cl::opt<bool> ClDumpStrictInstructions(
302 "msan-dump-strict-instructions",
303 cl::desc("print out instructions with default strict semantics"),
304 cl::Hidden, cl::init(false));
305
306static cl::opt<int> ClInstrumentationWithCallThreshold(
307 "msan-instrumentation-with-call-threshold",
308 cl::desc(
309 "If the function being instrumented requires more than "
310 "this number of checks and origin stores, use callbacks instead of "
311 "inline checks (-1 means never use callbacks)."),
312 cl::Hidden, cl::init(3500));
313
314static cl::opt<bool>
315 ClEnableKmsan("msan-kernel",
316 cl::desc("Enable KernelMemorySanitizer instrumentation"),
317 cl::Hidden, cl::init(false));
318
319static cl::opt<bool>
320 ClDisableChecks("msan-disable-checks",
321 cl::desc("Apply no_sanitize to the whole file"), cl::Hidden,
322 cl::init(false));
323
324static cl::opt<bool>
325 ClCheckConstantShadow("msan-check-constant-shadow",
326 cl::desc("Insert checks for constant shadow values"),
327 cl::Hidden, cl::init(true));
328
329// This is off by default because of a bug in gold:
330// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
331static cl::opt<bool>
332 ClWithComdat("msan-with-comdat",
333 cl::desc("Place MSan constructors in comdat sections"),
334 cl::Hidden, cl::init(false));
335
336// These options allow to specify custom memory map parameters
337// See MemoryMapParams for details.
338static cl::opt<uint64_t> ClAndMask("msan-and-mask",
339 cl::desc("Define custom MSan AndMask"),
340 cl::Hidden, cl::init(0));
341
342static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
343 cl::desc("Define custom MSan XorMask"),
344 cl::Hidden, cl::init(0));
345
346static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
347 cl::desc("Define custom MSan ShadowBase"),
348 cl::Hidden, cl::init(0));
349
350static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
351 cl::desc("Define custom MSan OriginBase"),
352 cl::Hidden, cl::init(0));
353
354const char kMsanModuleCtorName[] = "msan.module_ctor";
355const char kMsanInitName[] = "__msan_init";
356
357namespace {
358
359// Memory map parameters used in application-to-shadow address calculation.
360// Offset = (Addr & ~AndMask) ^ XorMask
361// Shadow = ShadowBase + Offset
362// Origin = OriginBase + Offset
363struct MemoryMapParams {
364 uint64_t AndMask;
365 uint64_t XorMask;
366 uint64_t ShadowBase;
367 uint64_t OriginBase;
368};
369
370struct PlatformMemoryMapParams {
371 const MemoryMapParams *bits32;
372 const MemoryMapParams *bits64;
373};
374
375} // end anonymous namespace
376
377// i386 Linux
378static const MemoryMapParams Linux_I386_MemoryMapParams = {
379 0x000080000000, // AndMask
380 0, // XorMask (not used)
381 0, // ShadowBase (not used)
382 0x000040000000, // OriginBase
383};
384
385// x86_64 Linux
386static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
387#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
388 0x400000000000, // AndMask
389 0, // XorMask (not used)
390 0, // ShadowBase (not used)
391 0x200000000000, // OriginBase
392#else
393 0, // AndMask (not used)
394 0x500000000000, // XorMask
395 0, // ShadowBase (not used)
396 0x100000000000, // OriginBase
397#endif
398};
399
400// mips64 Linux
401static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
402 0, // AndMask (not used)
403 0x008000000000, // XorMask
404 0, // ShadowBase (not used)
405 0x002000000000, // OriginBase
406};
407
408// ppc64 Linux
409static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
410 0xE00000000000, // AndMask
411 0x100000000000, // XorMask
412 0x080000000000, // ShadowBase
413 0x1C0000000000, // OriginBase
414};
415
416// s390x Linux
417static const MemoryMapParams Linux_S390X_MemoryMapParams = {
418 0xC00000000000, // AndMask
419 0, // XorMask (not used)
420 0x080000000000, // ShadowBase
421 0x1C0000000000, // OriginBase
422};
423
424// aarch64 Linux
425static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
426 0, // AndMask (not used)
427 0x06000000000, // XorMask
428 0, // ShadowBase (not used)
429 0x01000000000, // OriginBase
430};
431
432// aarch64 FreeBSD
433static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams = {
434 0x1800000000000, // AndMask
435 0x0400000000000, // XorMask
436 0x0200000000000, // ShadowBase
437 0x0700000000000, // OriginBase
438};
439
440// i386 FreeBSD
441static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
442 0x000180000000, // AndMask
443 0x000040000000, // XorMask
444 0x000020000000, // ShadowBase
445 0x000700000000, // OriginBase
446};
447
448// x86_64 FreeBSD
449static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
450 0xc00000000000, // AndMask
451 0x200000000000, // XorMask
452 0x100000000000, // ShadowBase
453 0x380000000000, // OriginBase
454};
455
456// x86_64 NetBSD
457static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
458 0, // AndMask
459 0x500000000000, // XorMask
460 0, // ShadowBase
461 0x100000000000, // OriginBase
462};
463
464static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
465 &Linux_I386_MemoryMapParams,
466 &Linux_X86_64_MemoryMapParams,
467};
468
469static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
470 nullptr,
471 &Linux_MIPS64_MemoryMapParams,
472};
473
474static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
475 nullptr,
476 &Linux_PowerPC64_MemoryMapParams,
477};
478
479static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
480 nullptr,
481 &Linux_S390X_MemoryMapParams,
482};
483
484static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
485 nullptr,
486 &Linux_AArch64_MemoryMapParams,
487};
488
489static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams = {
490 nullptr,
491 &FreeBSD_AArch64_MemoryMapParams,
492};
493
494static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
495 &FreeBSD_I386_MemoryMapParams,
496 &FreeBSD_X86_64_MemoryMapParams,
497};
498
499static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
500 nullptr,
501 &NetBSD_X86_64_MemoryMapParams,
502};
503
504namespace {
505
506/// Instrument functions of a module to detect uninitialized reads.
507///
508/// Instantiating MemorySanitizer inserts the msan runtime library API function
509/// declarations into the module if they don't exist already. Instantiating
510/// ensures the __msan_init function is in the list of global constructors for
511/// the module.
512class MemorySanitizer {
513public:
514 MemorySanitizer(Module &M, MemorySanitizerOptions Options)
515 : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
516 Recover(Options.Recover), EagerChecks(Options.EagerChecks) {
517 initializeModule(M);
518 }
519
520 // MSan cannot be moved or copied because of MapParams.
521 MemorySanitizer(MemorySanitizer &&) = delete;
522 MemorySanitizer &operator=(MemorySanitizer &&) = delete;
523 MemorySanitizer(const MemorySanitizer &) = delete;
524 MemorySanitizer &operator=(const MemorySanitizer &) = delete;
525
526 bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
527
528private:
529 friend struct MemorySanitizerVisitor;
530 friend struct VarArgAMD64Helper;
531 friend struct VarArgMIPS64Helper;
532 friend struct VarArgAArch64Helper;
533 friend struct VarArgPowerPC64Helper;
534 friend struct VarArgSystemZHelper;
535
536 void initializeModule(Module &M);
537 void initializeCallbacks(Module &M);
538 void createKernelApi(Module &M);
539 void createUserspaceApi(Module &M);
540
541 /// True if we're compiling the Linux kernel.
542 bool CompileKernel;
543 /// Track origins (allocation points) of uninitialized values.
544 int TrackOrigins;
545 bool Recover;
546 bool EagerChecks;
547
548 LLVMContext *C;
549 Type *IntptrTy;
550 Type *OriginTy;
551
552 // XxxTLS variables represent the per-thread state in MSan and per-task state
553 // in KMSAN.
554 // For the userspace these point to thread-local globals. In the kernel land
555 // they point to the members of a per-task struct obtained via a call to
556 // __msan_get_context_state().
557
558 /// Thread-local shadow storage for function parameters.
559 Value *ParamTLS;
560
561 /// Thread-local origin storage for function parameters.
562 Value *ParamOriginTLS;
563
564 /// Thread-local shadow storage for function return value.
565 Value *RetvalTLS;
566
567 /// Thread-local origin storage for function return value.
568 Value *RetvalOriginTLS;
569
570 /// Thread-local shadow storage for in-register va_arg function
571 /// parameters (x86_64-specific).
572 Value *VAArgTLS;
573
574 /// Thread-local shadow storage for in-register va_arg function
575 /// parameters (x86_64-specific).
576 Value *VAArgOriginTLS;
577
578 /// Thread-local shadow storage for va_arg overflow area
579 /// (x86_64-specific).
580 Value *VAArgOverflowSizeTLS;
581
582 /// Are the instrumentation callbacks set up?
583 bool CallbacksInitialized = false;
584
585 /// The run-time callback to print a warning.
586 FunctionCallee WarningFn;
587
588 // These arrays are indexed by log2(AccessSize).
589 FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
590 FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
591
592 /// Run-time helper that generates a new origin value for a stack
593 /// allocation.
594 FunctionCallee MsanSetAllocaOriginWithDescriptionFn;
595 // No description version
596 FunctionCallee MsanSetAllocaOriginNoDescriptionFn;
597
598 /// Run-time helper that poisons stack on function entry.
599 FunctionCallee MsanPoisonStackFn;
600
601 /// Run-time helper that records a store (or any event) of an
602 /// uninitialized value and returns an updated origin id encoding this info.
603 FunctionCallee MsanChainOriginFn;
604
605 /// Run-time helper that paints an origin over a region.
606 FunctionCallee MsanSetOriginFn;
607
608 /// MSan runtime replacements for memmove, memcpy and memset.
609 FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
610
611 /// KMSAN callback for task-local function argument shadow.
612 StructType *MsanContextStateTy;
613 FunctionCallee MsanGetContextStateFn;
614
615 /// Functions for poisoning/unpoisoning local variables
616 FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
617
618 /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
619 /// pointers.
620 FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
621 FunctionCallee MsanMetadataPtrForLoad_1_8[4];
622 FunctionCallee MsanMetadataPtrForStore_1_8[4];
623 FunctionCallee MsanInstrumentAsmStoreFn;
624
625 /// Helper to choose between different MsanMetadataPtrXxx().
626 FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
627
628 /// Memory map parameters used in application-to-shadow calculation.
629 const MemoryMapParams *MapParams;
630
631 /// Custom memory map parameters used when -msan-shadow-base or
632 // -msan-origin-base is provided.
633 MemoryMapParams CustomMapParams;
634
635 MDNode *ColdCallWeights;
636
637 /// Branch weights for origin store.
638 MDNode *OriginStoreWeights;
639};
640
641void insertModuleCtor(Module &M) {
642 getOrCreateSanitizerCtorAndInitFunctions(
643 M, kMsanModuleCtorName, kMsanInitName,
644 /*InitArgTypes=*/{},
645 /*InitArgs=*/{},
646 // This callback is invoked when the functions are created the first
647 // time. Hook them into the global ctors list in that case:
648 [&](Function *Ctor, FunctionCallee) {
649 if (!ClWithComdat) {
650 appendToGlobalCtors(M, Ctor, 0);
651 return;
652 }
653 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
654 Ctor->setComdat(MsanCtorComdat);
655 appendToGlobalCtors(M, Ctor, 0, Ctor);
656 });
657}
658
659template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
660 return (Opt.getNumOccurrences() > 0) ? Opt : Default;
661}
662
663} // end anonymous namespace
664
665MemorySanitizerOptions::MemorySanitizerOptions(int TO, bool R, bool K,
666 bool EagerChecks)
667 : Kernel(getOptOrDefault(ClEnableKmsan, K)),
668 TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
669 Recover(getOptOrDefault(ClKeepGoing, Kernel || R)),
670 EagerChecks(getOptOrDefault(ClEagerChecks, EagerChecks)) {}
671
672PreservedAnalyses MemorySanitizerPass::run(Function &F,
673 FunctionAnalysisManager &FAM) {
674 MemorySanitizer Msan(*F.getParent(), Options);
675 if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
676 return PreservedAnalyses::none();
677 return PreservedAnalyses::all();
678}
679
680PreservedAnalyses ModuleMemorySanitizerPass::run(Module &M,
681 ModuleAnalysisManager &AM) {
682 if (Options.Kernel)
683 return PreservedAnalyses::all();
684 insertModuleCtor(M);
685 return PreservedAnalyses::none();
686}
687
688void MemorySanitizerPass::printPipeline(
689 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
690 static_cast<PassInfoMixin<MemorySanitizerPass> *>(this)->printPipeline(
691 OS, MapClassName2PassName);
692 OS << "<";
693 if (Options.Recover)
694 OS << "recover;";
695 if (Options.Kernel)
696 OS << "kernel;";
697 if (Options.EagerChecks)
698 OS << "eager-checks;";
699 OS << "track-origins=" << Options.TrackOrigins;
700 OS << ">";
701}
702
703/// Create a non-const global initialized with the given string.
704///
705/// Creates a writable global for Str so that we can pass it to the
706/// run-time lib. Runtime uses first 4 bytes of the string to store the
707/// frame ID, so the string needs to be mutable.
708static GlobalVariable *createPrivateConstGlobalForString(Module &M,
709 StringRef Str) {
710 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
711 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/true,
712 GlobalValue::PrivateLinkage, StrConst, "");
713}
714
715/// Create KMSAN API callbacks.
716void MemorySanitizer::createKernelApi(Module &M) {
717 IRBuilder<> IRB(*C);
718
719 // These will be initialized in insertKmsanPrologue().
720 RetvalTLS = nullptr;
721 RetvalOriginTLS = nullptr;
722 ParamTLS = nullptr;
723 ParamOriginTLS = nullptr;
724 VAArgTLS = nullptr;
725 VAArgOriginTLS = nullptr;
726 VAArgOverflowSizeTLS = nullptr;
727
728 WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
729 IRB.getInt32Ty());
730 // Requests the per-task context state (kmsan_context_state*) from the
731 // runtime library.
732 MsanContextStateTy = StructType::get(
733 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
734 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
735 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
736 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
737 IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
738 OriginTy);
739 MsanGetContextStateFn = M.getOrInsertFunction(
740 "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
741
742 Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
743 PointerType::get(IRB.getInt32Ty(), 0));
744
745 for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
746 std::string name_load =
747 "__msan_metadata_ptr_for_load_" + std::to_string(size);
748 std::string name_store =
749 "__msan_metadata_ptr_for_store_" + std::to_string(size);
750 MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
751 name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
752 MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
753 name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
754 }
755
756 MsanMetadataPtrForLoadN = M.getOrInsertFunction(
757 "__msan_metadata_ptr_for_load_n", RetTy,
758 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
759 MsanMetadataPtrForStoreN = M.getOrInsertFunction(
760 "__msan_metadata_ptr_for_store_n", RetTy,
761 PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
762
763 // Functions for poisoning and unpoisoning memory.
764 MsanPoisonAllocaFn =
765 M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
766 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
767 MsanUnpoisonAllocaFn = M.getOrInsertFunction(
768 "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
769}
770
771static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
772 return M.getOrInsertGlobal(Name, Ty, [&] {
773 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
774 nullptr, Name, nullptr,
775 GlobalVariable::InitialExecTLSModel);
776 });
777}
778
779/// Insert declarations for userspace-specific functions and globals.
780void MemorySanitizer::createUserspaceApi(Module &M) {
781 IRBuilder<> IRB(*C);
782
783 // Create the callback.
784 // FIXME: this function should have "Cold" calling conv,
785 // which is not yet implemented.
786 if (TrackOrigins) {
787 StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
788 : "__msan_warning_with_origin_noreturn";
789 WarningFn =
790 M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), IRB.getInt32Ty());
791 } else {
792 StringRef WarningFnName =
793 Recover ? "__msan_warning" : "__msan_warning_noreturn";
794 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
795 }
796
797 // Create the global TLS variables.
798 RetvalTLS =
799 getOrInsertGlobal(M, "__msan_retval_tls",
800 ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
801
802 RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
803
804 ParamTLS =
805 getOrInsertGlobal(M, "__msan_param_tls",
806 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
807
808 ParamOriginTLS =
809 getOrInsertGlobal(M, "__msan_param_origin_tls",
810 ArrayType::get(OriginTy, kParamTLSSize / 4));
811
812 VAArgTLS =
813 getOrInsertGlobal(M, "__msan_va_arg_tls",
814 ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
815
816 VAArgOriginTLS =
817 getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
818 ArrayType::get(OriginTy, kParamTLSSize / 4));
819
820 VAArgOverflowSizeTLS =
821 getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
822
823 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
824 AccessSizeIndex++) {
825 unsigned AccessSize = 1 << AccessSizeIndex;
826 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
827 SmallVector<std::pair<unsigned, Attribute>, 2> MaybeWarningFnAttrs;
828 MaybeWarningFnAttrs.push_back(std::make_pair(
829 AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
830 MaybeWarningFnAttrs.push_back(std::make_pair(
831 AttributeList::FirstArgIndex + 1, Attribute::get(*C, Attribute::ZExt)));
832 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
833 FunctionName, AttributeList::get(*C, MaybeWarningFnAttrs),
834 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
835
836 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
837 SmallVector<std::pair<unsigned, Attribute>, 2> MaybeStoreOriginFnAttrs;
838 MaybeStoreOriginFnAttrs.push_back(std::make_pair(
839 AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
840 MaybeStoreOriginFnAttrs.push_back(std::make_pair(
841 AttributeList::FirstArgIndex + 2, Attribute::get(*C, Attribute::ZExt)));
842 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
843 FunctionName, AttributeList::get(*C, MaybeStoreOriginFnAttrs),
844 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(),
845 IRB.getInt32Ty());
846 }
847
848 MsanSetAllocaOriginWithDescriptionFn = M.getOrInsertFunction(
849 "__msan_set_alloca_origin_with_descr", IRB.getVoidTy(),
850 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy(), IRB.getInt8PtrTy());
851 MsanSetAllocaOriginNoDescriptionFn = M.getOrInsertFunction(
852 "__msan_set_alloca_origin_no_descr", IRB.getVoidTy(), IRB.getInt8PtrTy(),
853 IntptrTy, IRB.getInt8PtrTy());
854 MsanPoisonStackFn = M.getOrInsertFunction(
855 "__msan_poison_stack", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
856}
857
858/// Insert extern declaration of runtime-provided functions and globals.
859void MemorySanitizer::initializeCallbacks(Module &M) {
860 // Only do this once.
861 if (CallbacksInitialized)
862 return;
863
864 IRBuilder<> IRB(*C);
865 // Initialize callbacks that are common for kernel and userspace
866 // instrumentation.
867 MsanChainOriginFn = M.getOrInsertFunction("__msan_chain_origin",
868 IRB.getInt32Ty(), IRB.getInt32Ty());
869 MsanSetOriginFn =
870 M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
871 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
872 MemmoveFn =
873 M.getOrInsertFunction("__msan_memmove", IRB.getInt8PtrTy(),
874 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
875 MemcpyFn =
876 M.getOrInsertFunction("__msan_memcpy", IRB.getInt8PtrTy(),
877 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
878 MemsetFn =
879 M.getOrInsertFunction("__msan_memset", IRB.getInt8PtrTy(),
880 IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy);
881
882 MsanInstrumentAsmStoreFn =
883 M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
884 PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
885
886 if (CompileKernel) {
887 createKernelApi(M);
888 } else {
889 createUserspaceApi(M);
890 }
891 CallbacksInitialized = true;
892}
893
894FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
895 int size) {
896 FunctionCallee *Fns =
897 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
898 switch (size) {
899 case 1:
900 return Fns[0];
901 case 2:
902 return Fns[1];
903 case 4:
904 return Fns[2];
905 case 8:
906 return Fns[3];
907 default:
908 return nullptr;
909 }
910}
911
912/// Module-level initialization.
913///
914/// inserts a call to __msan_init to the module's constructor list.
915void MemorySanitizer::initializeModule(Module &M) {
916 auto &DL = M.getDataLayout();
917
918 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
919 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
920 // Check the overrides first
921 if (ShadowPassed || OriginPassed) {
922 CustomMapParams.AndMask = ClAndMask;
923 CustomMapParams.XorMask = ClXorMask;
924 CustomMapParams.ShadowBase = ClShadowBase;
925 CustomMapParams.OriginBase = ClOriginBase;
926 MapParams = &CustomMapParams;
927 } else {
928 Triple TargetTriple(M.getTargetTriple());
929 switch (TargetTriple.getOS()) {
930 case Triple::FreeBSD:
931 switch (TargetTriple.getArch()) {
932 case Triple::aarch64:
933 MapParams = FreeBSD_ARM_MemoryMapParams.bits64;
934 break;
935 case Triple::x86_64:
936 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
937 break;
938 case Triple::x86:
939 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
940 break;
941 default:
942 report_fatal_error("unsupported architecture");
943 }
944 break;
945 case Triple::NetBSD:
946 switch (TargetTriple.getArch()) {
947 case Triple::x86_64:
948 MapParams = NetBSD_X86_MemoryMapParams.bits64;
949 break;
950 default:
951 report_fatal_error("unsupported architecture");
952 }
953 break;
954 case Triple::Linux:
955 switch (TargetTriple.getArch()) {
956 case Triple::x86_64:
957 MapParams = Linux_X86_MemoryMapParams.bits64;
958 break;
959 case Triple::x86:
960 MapParams = Linux_X86_MemoryMapParams.bits32;
961 break;
962 case Triple::mips64:
963 case Triple::mips64el:
964 MapParams = Linux_MIPS_MemoryMapParams.bits64;
965 break;
966 case Triple::ppc64:
967 case Triple::ppc64le:
968 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
969 break;
970 case Triple::systemz:
971 MapParams = Linux_S390_MemoryMapParams.bits64;
972 break;
973 case Triple::aarch64:
974 case Triple::aarch64_be:
975 MapParams = Linux_ARM_MemoryMapParams.bits64;
976 break;
977 default:
978 report_fatal_error("unsupported architecture");
979 }
980 break;
981 default:
982 report_fatal_error("unsupported operating system");
983 }
984 }
985
986 C = &(M.getContext());
987 IRBuilder<> IRB(*C);
988 IntptrTy = IRB.getIntPtrTy(DL);
989 OriginTy = IRB.getInt32Ty();
990
991 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
992 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
993
994 if (!CompileKernel) {
995 if (TrackOrigins)
996 M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
997 return new GlobalVariable(
998 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
999 IRB.getInt32(TrackOrigins), "__msan_track_origins");
1000 });
1001
1002 if (Recover)
1003 M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
1004 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1005 GlobalValue::WeakODRLinkage,
1006 IRB.getInt32(Recover), "__msan_keep_going");
1007 });
1008 }
1009}
1010
1011namespace {
1012
1013/// A helper class that handles instrumentation of VarArg
1014/// functions on a particular platform.
1015///
1016/// Implementations are expected to insert the instrumentation
1017/// necessary to propagate argument shadow through VarArg function
1018/// calls. Visit* methods are called during an InstVisitor pass over
1019/// the function, and should avoid creating new basic blocks. A new
1020/// instance of this class is created for each instrumented function.
1021struct VarArgHelper {
1022 virtual ~VarArgHelper() = default;
1023
1024 /// Visit a CallBase.
1025 virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1026
1027 /// Visit a va_start call.
1028 virtual void visitVAStartInst(VAStartInst &I) = 0;
1029
1030 /// Visit a va_copy call.
1031 virtual void visitVACopyInst(VACopyInst &I) = 0;
1032
1033 /// Finalize function instrumentation.
1034 ///
1035 /// This method is called after visiting all interesting (see above)
1036 /// instructions in a function.
1037 virtual void finalizeInstrumentation() = 0;
1038};
1039
1040struct MemorySanitizerVisitor;
1041
1042} // end anonymous namespace
1043
1044static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1045 MemorySanitizerVisitor &Visitor);
1046
1047static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1048 if (TypeSize <= 8)
1049 return 0;
1050 return Log2_32_Ceil((TypeSize + 7) / 8);
1051}
1052
1053namespace {
1054
1055/// Helper class to attach debug information of the given instruction onto new
1056/// instructions inserted after.
1057class NextNodeIRBuilder : public IRBuilder<> {
1058public:
1059 explicit NextNodeIRBuilder(Instruction *IP) : IRBuilder<>(IP->getNextNode()) {
1060 SetCurrentDebugLocation(IP->getDebugLoc());
1061 }
1062};
1063
1064/// This class does all the work for a given function. Store and Load
1065/// instructions store and load corresponding shadow and origin
1066/// values. Most instructions propagate shadow from arguments to their
1067/// return values. Certain instructions (most importantly, BranchInst)
1068/// test their argument shadow and print reports (with a runtime call) if it's
1069/// non-zero.
1070struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1071 Function &F;
1072 MemorySanitizer &MS;
1073 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1074 ValueMap<Value *, Value *> ShadowMap, OriginMap;
1075 std::unique_ptr<VarArgHelper> VAHelper;
1076 const TargetLibraryInfo *TLI;
1077 Instruction *FnPrologueEnd;
1078
1079 // The following flags disable parts of MSan instrumentation based on
1080 // exclusion list contents and command-line options.
1081 bool InsertChecks;
1082 bool PropagateShadow;
1083 bool PoisonStack;
1084 bool PoisonUndef;
1085
1086 struct ShadowOriginAndInsertPoint {
1087 Value *Shadow;
1088 Value *Origin;
1089 Instruction *OrigIns;
1090
1091 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1092 : Shadow(S), Origin(O), OrigIns(I) {}
1093 };
1094 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1095 bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1096 SmallSetVector<AllocaInst *, 16> AllocaSet;
1097 SmallVector<std::pair<IntrinsicInst *, AllocaInst *>, 16> LifetimeStartList;
1098 SmallVector<StoreInst *, 16> StoreList;
1099
1100 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1101 const TargetLibraryInfo &TLI)
1102 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1103 bool SanitizeFunction =
1104 F.hasFnAttribute(Attribute::SanitizeMemory) && !ClDisableChecks;
1105 InsertChecks = SanitizeFunction;
1106 PropagateShadow = SanitizeFunction;
1107 PoisonStack = SanitizeFunction && ClPoisonStack;
1108 PoisonUndef = SanitizeFunction && ClPoisonUndef;
1109
1110 // In the presence of unreachable blocks, we may see Phi nodes with
1111 // incoming nodes from such blocks. Since InstVisitor skips unreachable
1112 // blocks, such nodes will not have any shadow value associated with them.
1113 // It's easier to remove unreachable blocks than deal with missing shadow.
1114 removeUnreachableBlocks(F);
1115
1116 MS.initializeCallbacks(*F.getParent());
1117 FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
1118 .CreateIntrinsic(Intrinsic::donothing, {}, {});
1119
1120 if (MS.CompileKernel) {
1121 IRBuilder<> IRB(FnPrologueEnd);
1122 insertKmsanPrologue(IRB);
1123 }
1124
1125 LLVM_DEBUG(if (!InsertChecks) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1126 << "MemorySanitizer is not inserting checks into '"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
1127 << F.getName() << "'\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { if (!InsertChecks) dbgs() << "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n"; } } while (false)
;
1128 }
1129
1130 bool isInPrologue(Instruction &I) {
1131 return I.getParent() == FnPrologueEnd->getParent() &&
1132 (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
1133 }
1134
1135 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1136 if (MS.TrackOrigins <= 1)
1137 return V;
1138 return IRB.CreateCall(MS.MsanChainOriginFn, V);
1139 }
1140
1141 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1142 const DataLayout &DL = F.getParent()->getDataLayout();
1143 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1144 if (IntptrSize == kOriginSize)
1145 return Origin;
1146 assert(IntptrSize == kOriginSize * 2)(static_cast <bool> (IntptrSize == kOriginSize * 2) ? void
(0) : __assert_fail ("IntptrSize == kOriginSize * 2", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1146, __extension__ __PRETTY_FUNCTION__))
;
1147 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1148 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1149 }
1150
1151 /// Fill memory range with the given origin value.
1152 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1153 unsigned Size, Align Alignment) {
1154 const DataLayout &DL = F.getParent()->getDataLayout();
1155 const Align IntptrAlignment = DL.getABITypeAlign(MS.IntptrTy);
1156 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1157 assert(IntptrAlignment >= kMinOriginAlignment)(static_cast <bool> (IntptrAlignment >= kMinOriginAlignment
) ? void (0) : __assert_fail ("IntptrAlignment >= kMinOriginAlignment"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1157
, __extension__ __PRETTY_FUNCTION__))
;
1158 assert(IntptrSize >= kOriginSize)(static_cast <bool> (IntptrSize >= kOriginSize) ? void
(0) : __assert_fail ("IntptrSize >= kOriginSize", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1158, __extension__ __PRETTY_FUNCTION__))
;
1159
1160 unsigned Ofs = 0;
1161 Align CurrentAlignment = Alignment;
1162 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1163 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1164 Value *IntptrOriginPtr =
1165 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1166 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1167 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1168 : IntptrOriginPtr;
1169 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1170 Ofs += IntptrSize / kOriginSize;
1171 CurrentAlignment = IntptrAlignment;
1172 }
1173 }
1174
1175 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1176 Value *GEP =
1177 i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1178 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1179 CurrentAlignment = kMinOriginAlignment;
1180 }
1181 }
1182
1183 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1184 Value *OriginPtr, Align Alignment, bool AsCall) {
1185 const DataLayout &DL = F.getParent()->getDataLayout();
1186 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1187 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1188 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1189 if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1190 if (!ClCheckConstantShadow || ConstantShadow->isZeroValue()) {
1191 // Origin is not needed: value is initialized or const shadow is
1192 // ignored.
1193 return;
1194 }
1195 if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
1196 // Copy origin as the value is definitely uninitialized.
1197 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1198 OriginAlignment);
1199 return;
1200 }
1201 // Fallback to runtime check, which still can be optimized out later.
1202 }
1203
1204 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1205 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1206 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1207 FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1208 Value *ConvertedShadow2 =
1209 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1210 CallBase *CB = IRB.CreateCall(
1211 Fn, {ConvertedShadow2,
1212 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
1213 CB->addParamAttr(0, Attribute::ZExt);
1214 CB->addParamAttr(2, Attribute::ZExt);
1215 } else {
1216 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1217 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1218 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1219 IRBuilder<> IRBNew(CheckTerm);
1220 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1221 OriginAlignment);
1222 }
1223 }
1224
1225 void materializeStores(bool InstrumentWithCalls) {
1226 for (StoreInst *SI : StoreList) {
1227 IRBuilder<> IRB(SI);
1228 Value *Val = SI->getValueOperand();
1229 Value *Addr = SI->getPointerOperand();
1230 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1231 Value *ShadowPtr, *OriginPtr;
1232 Type *ShadowTy = Shadow->getType();
1233 const Align Alignment = SI->getAlign();
1234 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1235 std::tie(ShadowPtr, OriginPtr) =
1236 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1237
1238 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1239 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " STORE: " << *NewSI <<
"\n"; } } while (false)
;
1240 (void)NewSI;
1241
1242 if (SI->isAtomic())
1243 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1244
1245 if (MS.TrackOrigins && !SI->isAtomic())
1246 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1247 OriginAlignment, InstrumentWithCalls);
1248 }
1249 }
1250
1251 /// Helper function to insert a warning at IRB's current insert point.
1252 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1253 if (!Origin)
1254 Origin = (Value *)IRB.getInt32(0);
1255 assert(Origin->getType()->isIntegerTy())(static_cast <bool> (Origin->getType()->isIntegerTy
()) ? void (0) : __assert_fail ("Origin->getType()->isIntegerTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1255
, __extension__ __PRETTY_FUNCTION__))
;
1256 if (MS.CompileKernel || MS.TrackOrigins)
1257 IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1258 else
1259 IRB.CreateCall(MS.WarningFn)->setCannotMerge();
1260 // FIXME: Insert UnreachableInst if !MS.Recover?
1261 // This may invalidate some of the following checks and needs to be done
1262 // at the very end.
1263 }
1264
1265 void materializeOneCheck(IRBuilder<> &IRB, Value *ConvertedShadow,
1266 Value *Origin, bool AsCall) {
1267 const DataLayout &DL = F.getParent()->getDataLayout();
1268 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1269 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1270 if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1271 FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1272 Value *ConvertedShadow2 =
1273 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1274 CallBase *CB = IRB.CreateCall(
1275 Fn, {ConvertedShadow2,
1276 MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
1277 CB->addParamAttr(0, Attribute::ZExt);
1278 CB->addParamAttr(1, Attribute::ZExt);
1279 } else {
1280 Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1281 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
1282 Cmp, &*IRB.GetInsertPoint(),
1283 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1284
1285 IRB.SetInsertPoint(CheckTerm);
1286 insertWarningFn(IRB, Origin);
1287 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CHECK: " << *Cmp <<
"\n"; } } while (false)
;
1288 }
1289 }
1290
1291 void materializeInstructionChecks(
1292 bool InstrumentWithCalls,
1293 ArrayRef<ShadowOriginAndInsertPoint> InstructionChecks) {
1294 const DataLayout &DL = F.getParent()->getDataLayout();
1295 // Disable combining in some cases. TrackOrigins checks each shadow to pick
1296 // correct origin. InstrumentWithCalls expects to reduce shadow using API.
1297 bool Combine = !InstrumentWithCalls && !MS.TrackOrigins;
1298 Instruction *Instruction = InstructionChecks.front().OrigIns;
1299 Value *Shadow = nullptr;
1300 for (const auto &ShadowData : InstructionChecks) {
1301 assert(ShadowData.OrigIns == Instruction)(static_cast <bool> (ShadowData.OrigIns == Instruction)
? void (0) : __assert_fail ("ShadowData.OrigIns == Instruction"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1301
, __extension__ __PRETTY_FUNCTION__))
;
1302 IRBuilder<> IRB(Instruction);
1303
1304 LLVM_DEBUG(dbgs() << " SHAD0 : " << *ShadowData.Shadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD0 : " << *ShadowData
.Shadow << "\n"; } } while (false)
;
1305 Value *ConvertedShadow = convertShadowToScalar(ShadowData.Shadow, IRB);
1306 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " SHAD1 : " << *ConvertedShadow
<< "\n"; } } while (false)
;
1307
1308 if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1309 if (!ClCheckConstantShadow || ConstantShadow->isZeroValue()) {
1310 // Skip, value is initialized or const shadow is ignored.
1311 continue;
1312 }
1313 if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
1314 // Report as the value is definitely uninitialized.
1315 insertWarningFn(IRB, ShadowData.Origin);
1316 if (!MS.Recover)
1317 return; // Always fail and stop here, not need to check the rest.
1318 // Skip entire instruction,
1319 continue;
1320 }
1321 // Fallback to runtime check, which still can be optimized out later.
1322 }
1323
1324 if (!Combine) {
1325 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin,
1326 InstrumentWithCalls);
1327 continue;
1328 }
1329
1330 Value *BoolShadow = convertToBool(ConvertedShadow, IRB, "_mscmp");
1331 Shadow = Shadow ? IRB.CreateOr(Shadow, BoolShadow, "_msor") : BoolShadow;
1332 }
1333
1334 if (Shadow) {
1335 assert(Combine)(static_cast <bool> (Combine) ? void (0) : __assert_fail
("Combine", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1335, __extension__ __PRETTY_FUNCTION__))
;
1336 IRBuilder<> IRB(Instruction);
1337 materializeOneCheck(IRB, Shadow, nullptr, false);
1338 }
1339 }
1340
1341 void materializeChecks(bool InstrumentWithCalls) {
1342 llvm::stable_sort(InstrumentationList,
1343 [](const ShadowOriginAndInsertPoint &L,
1344 const ShadowOriginAndInsertPoint &R) {
1345 return L.OrigIns < R.OrigIns;
1346 });
1347
1348 for (auto I = InstrumentationList.begin();
1349 I != InstrumentationList.end();) {
1350 auto J =
1351 std::find_if(I + 1, InstrumentationList.end(),
1352 [L = I->OrigIns](const ShadowOriginAndInsertPoint &R) {
1353 return L != R.OrigIns;
1354 });
1355 // Process all checks of instruction at once.
1356 materializeInstructionChecks(InstrumentWithCalls,
1357 ArrayRef<ShadowOriginAndInsertPoint>(I, J));
1358 I = J;
1359 }
1360
1361 LLVM_DEBUG(dbgs() << "DONE:\n" << F)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DONE:\n" << F; } } while (
false)
;
1362 }
1363
1364 // Returns the last instruction in the new prologue
1365 void insertKmsanPrologue(IRBuilder<> &IRB) {
1366 Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1367 Constant *Zero = IRB.getInt32(0);
1368 MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1369 {Zero, IRB.getInt32(0)}, "param_shadow");
1370 MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1371 {Zero, IRB.getInt32(1)}, "retval_shadow");
1372 MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1373 {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1374 MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1375 {Zero, IRB.getInt32(3)}, "va_arg_origin");
1376 MS.VAArgOverflowSizeTLS =
1377 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1378 {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1379 MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1380 {Zero, IRB.getInt32(5)}, "param_origin");
1381 MS.RetvalOriginTLS =
1382 IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1383 {Zero, IRB.getInt32(6)}, "retval_origin");
1384 }
1385
1386 /// Add MemorySanitizer instrumentation to a function.
1387 bool runOnFunction() {
1388 // Iterate all BBs in depth-first order and create shadow instructions
1389 // for all instructions (where applicable).
1390 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1391 for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
1392 visit(*BB);
1393
1394 // Finalize PHI nodes.
1395 for (PHINode *PN : ShadowPHINodes) {
1396 PHINode *PNS = cast<PHINode>(getShadow(PN));
1397 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1398 size_t NumValues = PN->getNumIncomingValues();
1399 for (size_t v = 0; v < NumValues; v++) {
1400 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1401 if (PNO)
1402 PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1403 }
1404 }
1405
1406 VAHelper->finalizeInstrumentation();
1407
1408 // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1409 // instrumenting only allocas.
1410 if (InstrumentLifetimeStart) {
1411 for (auto Item : LifetimeStartList) {
1412 instrumentAlloca(*Item.second, Item.first);
1413 AllocaSet.remove(Item.second);
1414 }
1415 }
1416 // Poison the allocas for which we didn't instrument the corresponding
1417 // lifetime intrinsics.
1418 for (AllocaInst *AI : AllocaSet)
1419 instrumentAlloca(*AI);
1420
1421 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1422 InstrumentationList.size() + StoreList.size() >
1423 (unsigned)ClInstrumentationWithCallThreshold;
1424
1425 // Insert shadow value checks.
1426 materializeChecks(InstrumentWithCalls);
1427
1428 // Delayed instrumentation of StoreInst.
1429 // This may not add new address checks.
1430 materializeStores(InstrumentWithCalls);
1431
1432 return true;
1433 }
1434
1435 /// Compute the shadow type that corresponds to a given Value.
1436 Type *getShadowTy(Value *V) { return getShadowTy(V->getType()); }
1437
1438 /// Compute the shadow type that corresponds to a given Type.
1439 Type *getShadowTy(Type *OrigTy) {
1440 if (!OrigTy->isSized()) {
1441 return nullptr;
1442 }
1443 // For integer type, shadow is the same as the original type.
1444 // This may return weird-sized types like i1.
1445 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1446 return IT;
1447 const DataLayout &DL = F.getParent()->getDataLayout();
1448 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1449 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1450 return FixedVectorType::get(IntegerType::get(*MS.C, EltSize),
1451 cast<FixedVectorType>(VT)->getNumElements());
1452 }
1453 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1454 return ArrayType::get(getShadowTy(AT->getElementType()),
1455 AT->getNumElements());
1456 }
1457 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1458 SmallVector<Type *, 4> Elements;
1459 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1460 Elements.push_back(getShadowTy(ST->getElementType(i)));
1461 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1462 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "getShadowTy: " << *ST <<
" ===> " << *Res << "\n"; } } while (false)
;
1463 return Res;
1464 }
1465 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1466 return IntegerType::get(*MS.C, TypeSize);
1467 }
1468
1469 /// Flatten a vector type.
1470 Type *getShadowTyNoVec(Type *ty) {
1471 if (VectorType *vt = dyn_cast<VectorType>(ty))
1472 return IntegerType::get(*MS.C,
1473 vt->getPrimitiveSizeInBits().getFixedSize());
1474 return ty;
1475 }
1476
1477 /// Extract combined shadow of struct elements as a bool
1478 Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1479 IRBuilder<> &IRB) {
1480 Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
1481 Value *Aggregator = FalseVal;
1482
1483 for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1484 // Combine by ORing together each element's bool shadow
1485 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1486 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1487 Value *ShadowBool = convertToBool(ShadowInner, IRB);
1488
1489 if (Aggregator != FalseVal)
1490 Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
1491 else
1492 Aggregator = ShadowBool;
1493 }
1494
1495 return Aggregator;
1496 }
1497
1498 // Extract combined shadow of array elements
1499 Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1500 IRBuilder<> &IRB) {
1501 if (!Array->getNumElements())
1502 return IRB.getIntN(/* width */ 1, /* value */ 0);
1503
1504 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1505 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1506
1507 for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1508 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1509 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1510 Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1511 }
1512 return Aggregator;
1513 }
1514
1515 /// Convert a shadow value to it's flattened variant. The resulting
1516 /// shadow may not necessarily have the same bit width as the input
1517 /// value, but it will always be comparable to zero.
1518 Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1519 if (StructType *Struct = dyn_cast<StructType>(V->getType()))
1520 return collapseStructShadow(Struct, V, IRB);
1521 if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
1522 return collapseArrayShadow(Array, V, IRB);
1523 Type *Ty = V->getType();
1524 Type *NoVecTy = getShadowTyNoVec(Ty);
1525 if (Ty == NoVecTy)
1526 return V;
1527 return IRB.CreateBitCast(V, NoVecTy);
1528 }
1529
1530 // Convert a scalar value to an i1 by comparing with 0
1531 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1532 Type *VTy = V->getType();
1533 assert(VTy->isIntegerTy())(static_cast <bool> (VTy->isIntegerTy()) ? void (0) :
__assert_fail ("VTy->isIntegerTy()", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1533, __extension__ __PRETTY_FUNCTION__))
;
1534 if (VTy->getIntegerBitWidth() == 1)
1535 // Just converting a bool to a bool, so do nothing.
1536 return V;
1537 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
1538 }
1539
1540 /// Compute the integer shadow offset that corresponds to a given
1541 /// application address.
1542 ///
1543 /// Offset = (Addr & ~AndMask) ^ XorMask
1544 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1545 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1546
1547 uint64_t AndMask = MS.MapParams->AndMask;
1548 if (AndMask)
1549 OffsetLong =
1550 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1551
1552 uint64_t XorMask = MS.MapParams->XorMask;
1553 if (XorMask)
1554 OffsetLong =
1555 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1556 return OffsetLong;
1557 }
1558
1559 /// Compute the shadow and origin addresses corresponding to a given
1560 /// application address.
1561 ///
1562 /// Shadow = ShadowBase + Offset
1563 /// Origin = (OriginBase + Offset) & ~3ULL
1564 std::pair<Value *, Value *>
1565 getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1566 MaybeAlign Alignment) {
1567 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1568 Value *ShadowLong = ShadowOffset;
1569 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1570 if (ShadowBase != 0) {
1571 ShadowLong =
1572 IRB.CreateAdd(ShadowLong, ConstantInt::get(MS.IntptrTy, ShadowBase));
1573 }
1574 Value *ShadowPtr =
1575 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1576 Value *OriginPtr = nullptr;
1577 if (MS.TrackOrigins) {
1578 Value *OriginLong = ShadowOffset;
1579 uint64_t OriginBase = MS.MapParams->OriginBase;
1580 if (OriginBase != 0)
1581 OriginLong = IRB.CreateAdd(OriginLong,
1582 ConstantInt::get(MS.IntptrTy, OriginBase));
1583 if (!Alignment || *Alignment < kMinOriginAlignment) {
1584 uint64_t Mask = kMinOriginAlignment.value() - 1;
1585 OriginLong =
1586 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1587 }
1588 OriginPtr =
1589 IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1590 }
1591 return std::make_pair(ShadowPtr, OriginPtr);
1592 }
1593
1594 std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1595 IRBuilder<> &IRB,
1596 Type *ShadowTy,
1597 bool isStore) {
1598 Value *ShadowOriginPtrs;
1599 const DataLayout &DL = F.getParent()->getDataLayout();
1600 int Size = DL.getTypeStoreSize(ShadowTy);
1601
1602 FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1603 Value *AddrCast =
1604 IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
1605 if (Getter) {
1606 ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1607 } else {
1608 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1609 ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1610 : MS.MsanMetadataPtrForLoadN,
1611 {AddrCast, SizeVal});
1612 }
1613 Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1614 ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1615 Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1616
1617 return std::make_pair(ShadowPtr, OriginPtr);
1618 }
1619
1620 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1621 Type *ShadowTy,
1622 MaybeAlign Alignment,
1623 bool isStore) {
1624 if (MS.CompileKernel)
1625 return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1626 return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1627 }
1628
1629 /// Compute the shadow address for a given function argument.
1630 ///
1631 /// Shadow = ParamTLS+ArgOffset.
1632 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) {
1633 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1634 if (ArgOffset)
1635 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1636 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1637 "_msarg");
1638 }
1639
1640 /// Compute the origin address for a given function argument.
1641 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB, int ArgOffset) {
1642 if (!MS.TrackOrigins)
1643 return nullptr;
1644 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1645 if (ArgOffset)
1646 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1647 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1648 "_msarg_o");
1649 }
1650
1651 /// Compute the shadow address for a retval.
1652 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1653 return IRB.CreatePointerCast(MS.RetvalTLS,
1654 PointerType::get(getShadowTy(A), 0), "_msret");
1655 }
1656
1657 /// Compute the origin address for a retval.
1658 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1659 // We keep a single origin for the entire retval. Might be too optimistic.
1660 return MS.RetvalOriginTLS;
1661 }
1662
1663 /// Set SV to be the shadow value for V.
1664 void setShadow(Value *V, Value *SV) {
1665 assert(!ShadowMap.count(V) && "Values may only have one shadow")(static_cast <bool> (!ShadowMap.count(V) && "Values may only have one shadow"
) ? void (0) : __assert_fail ("!ShadowMap.count(V) && \"Values may only have one shadow\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1665
, __extension__ __PRETTY_FUNCTION__))
;
1666 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1667 }
1668
1669 /// Set Origin to be the origin value for V.
1670 void setOrigin(Value *V, Value *Origin) {
1671 if (!MS.TrackOrigins)
1672 return;
1673 assert(!OriginMap.count(V) && "Values may only have one origin")(static_cast <bool> (!OriginMap.count(V) && "Values may only have one origin"
) ? void (0) : __assert_fail ("!OriginMap.count(V) && \"Values may only have one origin\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1673
, __extension__ __PRETTY_FUNCTION__))
;
1674 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ORIGIN: " << *V << " ==> "
<< *Origin << "\n"; } } while (false)
;
1675 OriginMap[V] = Origin;
1676 }
1677
1678 Constant *getCleanShadow(Type *OrigTy) {
1679 Type *ShadowTy = getShadowTy(OrigTy);
15
Value assigned to 'DebugFlag', which participates in a condition later
1680 if (!ShadowTy)
16
Assuming 'ShadowTy' is null
17
Taking true branch
1681 return nullptr;
1682 return Constant::getNullValue(ShadowTy);
1683 }
1684
1685 /// Create a clean shadow value for a given value.
1686 ///
1687 /// Clean shadow (all zeroes) means all bits of the value are defined
1688 /// (initialized).
1689 Constant *getCleanShadow(Value *V) { return getCleanShadow(V->getType()); }
14
Calling 'MemorySanitizerVisitor::getCleanShadow'
18
Returning from 'MemorySanitizerVisitor::getCleanShadow'
1690
1691 /// Create a dirty shadow of a given shadow type.
1692 Constant *getPoisonedShadow(Type *ShadowTy) {
1693 assert(ShadowTy)(static_cast <bool> (ShadowTy) ? void (0) : __assert_fail
("ShadowTy", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1693, __extension__ __PRETTY_FUNCTION__))
;
1694 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1695 return Constant::getAllOnesValue(ShadowTy);
1696 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1697 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1698 getPoisonedShadow(AT->getElementType()));
1699 return ConstantArray::get(AT, Vals);
1700 }
1701 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1702 SmallVector<Constant *, 4> Vals;
1703 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1704 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1705 return ConstantStruct::get(ST, Vals);
1706 }
1707 llvm_unreachable("Unexpected shadow type")::llvm::llvm_unreachable_internal("Unexpected shadow type", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1707)
;
1708 }
1709
1710 /// Create a dirty shadow for a given value.
1711 Constant *getPoisonedShadow(Value *V) {
1712 Type *ShadowTy = getShadowTy(V);
1713 if (!ShadowTy)
1714 return nullptr;
1715 return getPoisonedShadow(ShadowTy);
1716 }
1717
1718 /// Create a clean (zero) origin.
1719 Value *getCleanOrigin() { return Constant::getNullValue(MS.OriginTy); }
1720
1721 /// Get the shadow value for a given Value.
1722 ///
1723 /// This function either returns the value set earlier with setShadow,
1724 /// or extracts if from ParamTLS (for function arguments).
1725 Value *getShadow(Value *V) {
1726 if (Instruction *I
9.1
'I' is null
= dyn_cast<Instruction>(V)) {
9
Assuming 'V' is not a 'CastReturnType'
10
Taking false branch
1727 if (!PropagateShadow || I->getMetadata(LLVMContext::MD_nosanitize))
1728 return getCleanShadow(V);
1729 // For instructions the shadow is already stored in the map.
1730 Value *Shadow = ShadowMap[V];
1731 if (!Shadow) {
1732 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "No shadow: " << *V <<
"\n" << *(I->getParent()); } } while (false)
;
1733 (void)I;
1734 assert(Shadow && "No shadow for a value")(static_cast <bool> (Shadow && "No shadow for a value"
) ? void (0) : __assert_fail ("Shadow && \"No shadow for a value\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1734
, __extension__ __PRETTY_FUNCTION__))
;
1735 }
1736 return Shadow;
1737 }
1738 if (UndefValue *U
11.1
'U' is non-null
= dyn_cast<UndefValue>(V)) {
11
Assuming 'V' is a 'CastReturnType'
1739 Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
12
Assuming field 'PropagateShadow' is false
20
'AllOnes' initialized to a null pointer value
1740 : getCleanShadow(V);
13
Calling 'MemorySanitizerVisitor::getCleanShadow'
19
Returning from 'MemorySanitizerVisitor::getCleanShadow'
1741 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Undef: " << *U << " ==> "
<< *AllOnes << "\n"; } } while (false)
;
21
Assuming 'DebugFlag' is true
22
Assuming the condition is true
23
Taking true branch
24
Forming reference to null pointer
1742 (void)U;
1743 return AllOnes;
1744 }
1745 if (Argument *A = dyn_cast<Argument>(V)) {
1746 // For arguments we compute the shadow on demand and store it in the map.
1747 Value *&ShadowPtr = ShadowMap[V];
1748 if (ShadowPtr)
1749 return ShadowPtr;
1750 Function *F = A->getParent();
1751 IRBuilder<> EntryIRB(FnPrologueEnd);
1752 unsigned ArgOffset = 0;
1753 const DataLayout &DL = F->getParent()->getDataLayout();
1754 for (auto &FArg : F->args()) {
1755 if (!FArg.getType()->isSized()) {
1756 LLVM_DEBUG(dbgs() << "Arg is not sized\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg is not sized\n"; } } while (
false)
;
1757 continue;
1758 }
1759
1760 unsigned Size = FArg.hasByValAttr()
1761 ? DL.getTypeAllocSize(FArg.getParamByValType())
1762 : DL.getTypeAllocSize(FArg.getType());
1763
1764 if (A == &FArg) {
1765 bool Overflow = ArgOffset + Size > kParamTLSSize;
1766 if (FArg.hasByValAttr()) {
1767 // ByVal pointer itself has clean shadow. We copy the actual
1768 // argument shadow to the underlying memory.
1769 // Figure out maximal valid memcpy alignment.
1770 const Align ArgAlign = DL.getValueOrABITypeAlignment(
1771 MaybeAlign(FArg.getParamAlignment()), FArg.getParamByValType());
1772 Value *CpShadowPtr, *CpOriginPtr;
1773 std::tie(CpShadowPtr, CpOriginPtr) =
1774 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1775 /*isStore*/ true);
1776 if (!PropagateShadow || Overflow) {
1777 // ParamTLS overflow.
1778 EntryIRB.CreateMemSet(
1779 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1780 Size, ArgAlign);
1781 } else {
1782 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1783 const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1784 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1785 CopyAlign, Size);
1786 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ByValCpy: " << *Cpy <<
"\n"; } } while (false)
;
1787 (void)Cpy;
1788
1789 if (MS.TrackOrigins) {
1790 Value *OriginPtr =
1791 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1792 // FIXME: OriginSize should be:
1793 // alignTo(V % kMinOriginAlignment + Size, kMinOriginAlignment)
1794 unsigned OriginSize = alignTo(Size, kMinOriginAlignment);
1795 EntryIRB.CreateMemCpy(
1796 CpOriginPtr,
1797 /* by getShadowOriginPtr */ kMinOriginAlignment, OriginPtr,
1798 /* by origin_tls[ArgOffset] */ kMinOriginAlignment,
1799 OriginSize);
1800 }
1801 }
1802 }
1803
1804 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
1805 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
1806 ShadowPtr = getCleanShadow(V);
1807 setOrigin(A, getCleanOrigin());
1808 } else {
1809 // Shadow over TLS
1810 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1811 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1812 kShadowTLSAlignment);
1813 if (MS.TrackOrigins) {
1814 Value *OriginPtr =
1815 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1816 setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1817 }
1818 }
1819 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << *ShadowPtr << "\n"; } } while (false
)
1820 << " ARG: " << FArg << " ==> " << *ShadowPtr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ARG: " << FArg <<
" ==> " << *ShadowPtr << "\n"; } } while (false
)
;
1821 break;
1822 }
1823
1824 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1825 }
1826 assert(ShadowPtr && "Could not find shadow for an argument")(static_cast <bool> (ShadowPtr && "Could not find shadow for an argument"
) ? void (0) : __assert_fail ("ShadowPtr && \"Could not find shadow for an argument\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1826
, __extension__ __PRETTY_FUNCTION__))
;
1827 return ShadowPtr;
1828 }
1829 // For everything else the shadow is zero.
1830 return getCleanShadow(V);
1831 }
1832
1833 /// Get the shadow for i-th argument of the instruction I.
1834 Value *getShadow(Instruction *I, int i) {
1835 return getShadow(I->getOperand(i));
1836 }
1837
1838 /// Get the origin for a value.
1839 Value *getOrigin(Value *V) {
1840 if (!MS.TrackOrigins)
1841 return nullptr;
1842 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
1843 return getCleanOrigin();
1844 assert((isa<Instruction>(V) || isa<Argument>(V)) &&(static_cast <bool> ((isa<Instruction>(V) || isa<
Argument>(V)) && "Unexpected value type in getOrigin()"
) ? void (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1845
, __extension__ __PRETTY_FUNCTION__))
1845 "Unexpected value type in getOrigin()")(static_cast <bool> ((isa<Instruction>(V) || isa<
Argument>(V)) && "Unexpected value type in getOrigin()"
) ? void (0) : __assert_fail ("(isa<Instruction>(V) || isa<Argument>(V)) && \"Unexpected value type in getOrigin()\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1845
, __extension__ __PRETTY_FUNCTION__))
;
1846 if (Instruction *I = dyn_cast<Instruction>(V)) {
1847 if (I->getMetadata(LLVMContext::MD_nosanitize))
1848 return getCleanOrigin();
1849 }
1850 Value *Origin = OriginMap[V];
1851 assert(Origin && "Missing origin")(static_cast <bool> (Origin && "Missing origin"
) ? void (0) : __assert_fail ("Origin && \"Missing origin\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1851
, __extension__ __PRETTY_FUNCTION__))
;
1852 return Origin;
1853 }
1854
1855 /// Get the origin for i-th argument of the instruction I.
1856 Value *getOrigin(Instruction *I, int i) {
1857 return getOrigin(I->getOperand(i));
1858 }
1859
1860 /// Remember the place where a shadow check should be inserted.
1861 ///
1862 /// This location will be later instrumented with a check that will print a
1863 /// UMR warning in runtime if the shadow value is not 0.
1864 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1865 assert(Shadow)(static_cast <bool> (Shadow) ? void (0) : __assert_fail
("Shadow", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1865, __extension__ __PRETTY_FUNCTION__))
;
1866 if (!InsertChecks)
1867 return;
1868#ifndef NDEBUG
1869 Type *ShadowTy = Shadow->getType();
1870 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||(static_cast <bool> ((isa<IntegerType>(ShadowTy) ||
isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy
) || isa<ArrayType>(ShadowTy)) && "Can only insert checks for integer, vector, and aggregate shadow "
"types") ? void (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) && \"Can only insert checks for integer, vector, and aggregate shadow \" \"types\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1873
, __extension__ __PRETTY_FUNCTION__))
1871 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&(static_cast <bool> ((isa<IntegerType>(ShadowTy) ||
isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy
) || isa<ArrayType>(ShadowTy)) && "Can only insert checks for integer, vector, and aggregate shadow "
"types") ? void (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) && \"Can only insert checks for integer, vector, and aggregate shadow \" \"types\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1873
, __extension__ __PRETTY_FUNCTION__))
1872 "Can only insert checks for integer, vector, and aggregate shadow "(static_cast <bool> ((isa<IntegerType>(ShadowTy) ||
isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy
) || isa<ArrayType>(ShadowTy)) && "Can only insert checks for integer, vector, and aggregate shadow "
"types") ? void (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) && \"Can only insert checks for integer, vector, and aggregate shadow \" \"types\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1873
, __extension__ __PRETTY_FUNCTION__))
1873 "types")(static_cast <bool> ((isa<IntegerType>(ShadowTy) ||
isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy
) || isa<ArrayType>(ShadowTy)) && "Can only insert checks for integer, vector, and aggregate shadow "
"types") ? void (0) : __assert_fail ("(isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) || isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) && \"Can only insert checks for integer, vector, and aggregate shadow \" \"types\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1873
, __extension__ __PRETTY_FUNCTION__))
;
1874#endif
1875 InstrumentationList.push_back(
1876 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1877 }
1878
1879 /// Remember the place where a shadow check should be inserted.
1880 ///
1881 /// This location will be later instrumented with a check that will print a
1882 /// UMR warning in runtime if the value is not fully defined.
1883 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1884 assert(Val)(static_cast <bool> (Val) ? void (0) : __assert_fail ("Val"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1884
, __extension__ __PRETTY_FUNCTION__))
;
1885 Value *Shadow, *Origin;
1886 if (ClCheckConstantShadow) {
1887 Shadow = getShadow(Val);
1888 if (!Shadow)
1889 return;
1890 Origin = getOrigin(Val);
1891 } else {
1892 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1893 if (!Shadow)
1894 return;
1895 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1896 }
1897 insertShadowCheck(Shadow, Origin, OrigIns);
1898 }
1899
1900 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1901 switch (a) {
1902 case AtomicOrdering::NotAtomic:
1903 return AtomicOrdering::NotAtomic;
1904 case AtomicOrdering::Unordered:
1905 case AtomicOrdering::Monotonic:
1906 case AtomicOrdering::Release:
1907 return AtomicOrdering::Release;
1908 case AtomicOrdering::Acquire:
1909 case AtomicOrdering::AcquireRelease:
1910 return AtomicOrdering::AcquireRelease;
1911 case AtomicOrdering::SequentiallyConsistent:
1912 return AtomicOrdering::SequentiallyConsistent;
1913 }
1914 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1914)
;
1915 }
1916
1917 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
1918 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1919 uint32_t OrderingTable[NumOrderings] = {};
1920
1921 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1922 OrderingTable[(int)AtomicOrderingCABI::release] =
1923 (int)AtomicOrderingCABI::release;
1924 OrderingTable[(int)AtomicOrderingCABI::consume] =
1925 OrderingTable[(int)AtomicOrderingCABI::acquire] =
1926 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1927 (int)AtomicOrderingCABI::acq_rel;
1928 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1929 (int)AtomicOrderingCABI::seq_cst;
1930
1931 return ConstantDataVector::get(IRB.getContext(),
1932 makeArrayRef(OrderingTable, NumOrderings));
1933 }
1934
1935 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1936 switch (a) {
1937 case AtomicOrdering::NotAtomic:
1938 return AtomicOrdering::NotAtomic;
1939 case AtomicOrdering::Unordered:
1940 case AtomicOrdering::Monotonic:
1941 case AtomicOrdering::Acquire:
1942 return AtomicOrdering::Acquire;
1943 case AtomicOrdering::Release:
1944 case AtomicOrdering::AcquireRelease:
1945 return AtomicOrdering::AcquireRelease;
1946 case AtomicOrdering::SequentiallyConsistent:
1947 return AtomicOrdering::SequentiallyConsistent;
1948 }
1949 llvm_unreachable("Unknown ordering")::llvm::llvm_unreachable_internal("Unknown ordering", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 1949)
;
1950 }
1951
1952 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
1953 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1954 uint32_t OrderingTable[NumOrderings] = {};
1955
1956 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1957 OrderingTable[(int)AtomicOrderingCABI::acquire] =
1958 OrderingTable[(int)AtomicOrderingCABI::consume] =
1959 (int)AtomicOrderingCABI::acquire;
1960 OrderingTable[(int)AtomicOrderingCABI::release] =
1961 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1962 (int)AtomicOrderingCABI::acq_rel;
1963 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1964 (int)AtomicOrderingCABI::seq_cst;
1965
1966 return ConstantDataVector::get(IRB.getContext(),
1967 makeArrayRef(OrderingTable, NumOrderings));
1968 }
1969
1970 // ------------------- Visitors.
1971 using InstVisitor<MemorySanitizerVisitor>::visit;
1972 void visit(Instruction &I) {
1973 if (I.getMetadata(LLVMContext::MD_nosanitize))
1974 return;
1975 // Don't want to visit if we're in the prologue
1976 if (isInPrologue(I))
1977 return;
1978 InstVisitor<MemorySanitizerVisitor>::visit(I);
1979 }
1980
1981 /// Instrument LoadInst
1982 ///
1983 /// Loads the corresponding shadow and (optionally) origin.
1984 /// Optionally, checks that the load address is fully defined.
1985 void visitLoadInst(LoadInst &I) {
1986 assert(I.getType()->isSized() && "Load type must have size")(static_cast <bool> (I.getType()->isSized() &&
"Load type must have size") ? void (0) : __assert_fail ("I.getType()->isSized() && \"Load type must have size\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1986
, __extension__ __PRETTY_FUNCTION__))
;
1987 assert(!I.getMetadata(LLVMContext::MD_nosanitize))(static_cast <bool> (!I.getMetadata(LLVMContext::MD_nosanitize
)) ? void (0) : __assert_fail ("!I.getMetadata(LLVMContext::MD_nosanitize)"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 1987
, __extension__ __PRETTY_FUNCTION__))
;
1988 NextNodeIRBuilder IRB(&I);
1989 Type *ShadowTy = getShadowTy(&I);
1990 Value *Addr = I.getPointerOperand();
1991 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
1992 const Align Alignment = I.getAlign();
1993 if (PropagateShadow) {
1994 std::tie(ShadowPtr, OriginPtr) =
1995 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1996 setShadow(&I,
1997 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
1998 } else {
1999 setShadow(&I, getCleanShadow(&I));
2000 }
2001
2002 if (ClCheckAccessAddress)
2003 insertShadowCheck(I.getPointerOperand(), &I);
2004
2005 if (I.isAtomic())
2006 I.setOrdering(addAcquireOrdering(I.getOrdering()));
2007
2008 if (MS.TrackOrigins) {
2009 if (PropagateShadow) {
2010 const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
2011 setOrigin(
2012 &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
2013 } else {
2014 setOrigin(&I, getCleanOrigin());
2015 }
2016 }
2017 }
2018
2019 /// Instrument StoreInst
2020 ///
2021 /// Stores the corresponding shadow and (optionally) origin.
2022 /// Optionally, checks that the store address is fully defined.
2023 void visitStoreInst(StoreInst &I) {
2024 StoreList.push_back(&I);
2025 if (ClCheckAccessAddress)
2026 insertShadowCheck(I.getPointerOperand(), &I);
2027 }
2028
2029 void handleCASOrRMW(Instruction &I) {
2030 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I))(static_cast <bool> (isa<AtomicRMWInst>(I) || isa
<AtomicCmpXchgInst>(I)) ? void (0) : __assert_fail ("isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I)"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2030
, __extension__ __PRETTY_FUNCTION__))
;
2031
2032 IRBuilder<> IRB(&I);
2033 Value *Addr = I.getOperand(0);
2034 Value *Val = I.getOperand(1);
2035 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, getShadowTy(Val), Align(1),
2036 /*isStore*/ true)
2037 .first;
2038
2039 if (ClCheckAccessAddress)
2040 insertShadowCheck(Addr, &I);
2041
2042 // Only test the conditional argument of cmpxchg instruction.
2043 // The other argument can potentially be uninitialized, but we can not
2044 // detect this situation reliably without possible false positives.
2045 if (isa<AtomicCmpXchgInst>(I))
2046 insertShadowCheck(Val, &I);
2047
2048 IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
2049
2050 setShadow(&I, getCleanShadow(&I));
2051 setOrigin(&I, getCleanOrigin());
2052 }
2053
2054 void visitAtomicRMWInst(AtomicRMWInst &I) {
2055 handleCASOrRMW(I);
2056 I.setOrdering(addReleaseOrdering(I.getOrdering()));
2057 }
2058
2059 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2060 handleCASOrRMW(I);
2061 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
2062 }
2063
2064 // Vector manipulation.
2065 void visitExtractElementInst(ExtractElementInst &I) {
2066 insertShadowCheck(I.getOperand(1), &I);
2067 IRBuilder<> IRB(&I);
2068 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
2069 "_msprop"));
2070 setOrigin(&I, getOrigin(&I, 0));
2071 }
2072
2073 void visitInsertElementInst(InsertElementInst &I) {
2074 insertShadowCheck(I.getOperand(2), &I);
2075 IRBuilder<> IRB(&I);
2076 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
2077 I.getOperand(2), "_msprop"));
2078 setOriginForNaryOp(I);
2079 }
2080
2081 void visitShuffleVectorInst(ShuffleVectorInst &I) {
2082 IRBuilder<> IRB(&I);
2083 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
2084 I.getShuffleMask(), "_msprop"));
2085 setOriginForNaryOp(I);
2086 }
2087
2088 // Casts.
2089 void visitSExtInst(SExtInst &I) {
2090 IRBuilder<> IRB(&I);
2091 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
2092 setOrigin(&I, getOrigin(&I, 0));
2093 }
2094
2095 void visitZExtInst(ZExtInst &I) {
2096 IRBuilder<> IRB(&I);
2097 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
2098 setOrigin(&I, getOrigin(&I, 0));
2099 }
2100
2101 void visitTruncInst(TruncInst &I) {
2102 IRBuilder<> IRB(&I);
2103 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
2104 setOrigin(&I, getOrigin(&I, 0));
2105 }
2106
2107 void visitBitCastInst(BitCastInst &I) {
2108 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2109 // a musttail call and a ret, don't instrument. New instructions are not
2110 // allowed after a musttail call.
2111 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
2112 if (CI->isMustTailCall())
2113 return;
2114 IRBuilder<> IRB(&I);
2115 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
2116 setOrigin(&I, getOrigin(&I, 0));
2117 }
2118
2119 void visitPtrToIntInst(PtrToIntInst &I) {
2120 IRBuilder<> IRB(&I);
2121 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2122 "_msprop_ptrtoint"));
2123 setOrigin(&I, getOrigin(&I, 0));
2124 }
2125
2126 void visitIntToPtrInst(IntToPtrInst &I) {
2127 IRBuilder<> IRB(&I);
2128 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2129 "_msprop_inttoptr"));
2130 setOrigin(&I, getOrigin(&I, 0));
2131 }
2132
2133 void visitFPToSIInst(CastInst &I) { handleShadowOr(I); }
2134 void visitFPToUIInst(CastInst &I) { handleShadowOr(I); }
2135 void visitSIToFPInst(CastInst &I) { handleShadowOr(I); }
2136 void visitUIToFPInst(CastInst &I) { handleShadowOr(I); }
2137 void visitFPExtInst(CastInst &I) { handleShadowOr(I); }
2138 void visitFPTruncInst(CastInst &I) { handleShadowOr(I); }
2139
2140 /// Propagate shadow for bitwise AND.
2141 ///
2142 /// This code is exact, i.e. if, for example, a bit in the left argument
2143 /// is defined and 0, then neither the value not definedness of the
2144 /// corresponding bit in B don't affect the resulting shadow.
2145 void visitAnd(BinaryOperator &I) {
2146 IRBuilder<> IRB(&I);
2147 // "And" of 0 and a poisoned value results in unpoisoned value.
2148 // 1&1 => 1; 0&1 => 0; p&1 => p;
2149 // 1&0 => 0; 0&0 => 0; p&0 => 0;
2150 // 1&p => p; 0&p => 0; p&p => p;
2151 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2152 Value *S1 = getShadow(&I, 0);
2153 Value *S2 = getShadow(&I, 1);
2154 Value *V1 = I.getOperand(0);
2155 Value *V2 = I.getOperand(1);
2156 if (V1->getType() != S1->getType()) {
2157 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2158 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2159 }
2160 Value *S1S2 = IRB.CreateAnd(S1, S2);
2161 Value *V1S2 = IRB.CreateAnd(V1, S2);
2162 Value *S1V2 = IRB.CreateAnd(S1, V2);
2163 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2164 setOriginForNaryOp(I);
2165 }
2166
2167 void visitOr(BinaryOperator &I) {
2168 IRBuilder<> IRB(&I);
2169 // "Or" of 1 and a poisoned value results in unpoisoned value.
2170 // 1|1 => 1; 0|1 => 1; p|1 => 1;
2171 // 1|0 => 1; 0|0 => 0; p|0 => p;
2172 // 1|p => 1; 0|p => p; p|p => p;
2173 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2174 Value *S1 = getShadow(&I, 0);
2175 Value *S2 = getShadow(&I, 1);
2176 Value *V1 = IRB.CreateNot(I.getOperand(0));
2177 Value *V2 = IRB.CreateNot(I.getOperand(1));
2178 if (V1->getType() != S1->getType()) {
2179 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2180 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2181 }
2182 Value *S1S2 = IRB.CreateAnd(S1, S2);
2183 Value *V1S2 = IRB.CreateAnd(V1, S2);
2184 Value *S1V2 = IRB.CreateAnd(S1, V2);
2185 setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2186 setOriginForNaryOp(I);
2187 }
2188
2189 /// Default propagation of shadow and/or origin.
2190 ///
2191 /// This class implements the general case of shadow propagation, used in all
2192 /// cases where we don't know and/or don't care about what the operation
2193 /// actually does. It converts all input shadow values to a common type
2194 /// (extending or truncating as necessary), and bitwise OR's them.
2195 ///
2196 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2197 /// fully initialized), and less prone to false positives.
2198 ///
2199 /// This class also implements the general case of origin propagation. For a
2200 /// Nary operation, result origin is set to the origin of an argument that is
2201 /// not entirely initialized. If there is more than one such arguments, the
2202 /// rightmost of them is picked. It does not matter which one is picked if all
2203 /// arguments are initialized.
2204 template <bool CombineShadow> class Combiner {
2205 Value *Shadow = nullptr;
2206 Value *Origin = nullptr;
2207 IRBuilder<> &IRB;
2208 MemorySanitizerVisitor *MSV;
2209
2210 public:
2211 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2212 : IRB(IRB), MSV(MSV) {}
2213
2214 /// Add a pair of shadow and origin values to the mix.
2215 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2216 if (CombineShadow) {
2217 assert(OpShadow)(static_cast <bool> (OpShadow) ? void (0) : __assert_fail
("OpShadow", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2217, __extension__ __PRETTY_FUNCTION__))
;
2218 if (!Shadow)
2219 Shadow = OpShadow;
2220 else {
2221 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2222 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2223 }
2224 }
2225
2226 if (MSV->MS.TrackOrigins) {
2227 assert(OpOrigin)(static_cast <bool> (OpOrigin) ? void (0) : __assert_fail
("OpOrigin", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2227, __extension__ __PRETTY_FUNCTION__))
;
2228 if (!Origin) {
2229 Origin = OpOrigin;
2230 } else {
2231 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2232 // No point in adding something that might result in 0 origin value.
2233 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2234 Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
2235 Value *Cond =
2236 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2237 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2238 }
2239 }
2240 }
2241 return *this;
2242 }
2243
2244 /// Add an application value to the mix.
2245 Combiner &Add(Value *V) {
2246 Value *OpShadow = MSV->getShadow(V);
2247 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2248 return Add(OpShadow, OpOrigin);
2249 }
2250
2251 /// Set the current combined values as the given instruction's shadow
2252 /// and origin.
2253 void Done(Instruction *I) {
2254 if (CombineShadow) {
2255 assert(Shadow)(static_cast <bool> (Shadow) ? void (0) : __assert_fail
("Shadow", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2255, __extension__ __PRETTY_FUNCTION__))
;
2256 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2257 MSV->setShadow(I, Shadow);
2258 }
2259 if (MSV->MS.TrackOrigins) {
2260 assert(Origin)(static_cast <bool> (Origin) ? void (0) : __assert_fail
("Origin", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2260, __extension__ __PRETTY_FUNCTION__))
;
2261 MSV->setOrigin(I, Origin);
2262 }
2263 }
2264 };
2265
2266 using ShadowAndOriginCombiner = Combiner<true>;
2267 using OriginCombiner = Combiner<false>;
2268
2269 /// Propagate origin for arbitrary operation.
2270 void setOriginForNaryOp(Instruction &I) {
2271 if (!MS.TrackOrigins)
2272 return;
2273 IRBuilder<> IRB(&I);
2274 OriginCombiner OC(this, IRB);
2275 for (Use &Op : I.operands())
2276 OC.Add(Op.get());
2277 OC.Done(&I);
2278 }
2279
2280 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2281 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&(static_cast <bool> (!(Ty->isVectorTy() && Ty
->getScalarType()->isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? void (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2282
, __extension__ __PRETTY_FUNCTION__))
2282 "Vector of pointers is not a valid shadow type")(static_cast <bool> (!(Ty->isVectorTy() && Ty
->getScalarType()->isPointerTy()) && "Vector of pointers is not a valid shadow type"
) ? void (0) : __assert_fail ("!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) && \"Vector of pointers is not a valid shadow type\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2282
, __extension__ __PRETTY_FUNCTION__))
;
2283 return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2284 Ty->getScalarSizeInBits()
2285 : Ty->getPrimitiveSizeInBits();
2286 }
2287
2288 /// Cast between two shadow types, extending or truncating as
2289 /// necessary.
2290 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2291 bool Signed = false) {
2292 Type *srcTy = V->getType();
2293 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2294 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2295 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2296 return IRB.CreateICmpNE(V, getCleanShadow(V));
2297
2298 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2299 return IRB.CreateIntCast(V, dstTy, Signed);
2300 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2301 cast<FixedVectorType>(dstTy)->getNumElements() ==
2302 cast<FixedVectorType>(srcTy)->getNumElements())
2303 return IRB.CreateIntCast(V, dstTy, Signed);
2304 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2305 Value *V2 =
2306 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2307 return IRB.CreateBitCast(V2, dstTy);
2308 // TODO: handle struct types.
2309 }
2310
2311 /// Cast an application value to the type of its own shadow.
2312 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2313 Type *ShadowTy = getShadowTy(V);
2314 if (V->getType() == ShadowTy)
2315 return V;
2316 if (V->getType()->isPtrOrPtrVectorTy())
2317 return IRB.CreatePtrToInt(V, ShadowTy);
2318 else
2319 return IRB.CreateBitCast(V, ShadowTy);
2320 }
2321
2322 /// Propagate shadow for arbitrary operation.
2323 void handleShadowOr(Instruction &I) {
2324 IRBuilder<> IRB(&I);
2325 ShadowAndOriginCombiner SC(this, IRB);
2326 for (Use &Op : I.operands())
2327 SC.Add(Op.get());
2328 SC.Done(&I);
2329 }
2330
2331 void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2332
2333 // Handle multiplication by constant.
2334 //
2335 // Handle a special case of multiplication by constant that may have one or
2336 // more zeros in the lower bits. This makes corresponding number of lower bits
2337 // of the result zero as well. We model it by shifting the other operand
2338 // shadow left by the required number of bits. Effectively, we transform
2339 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2340 // We use multiplication by 2**N instead of shift to cover the case of
2341 // multiplication by 0, which may occur in some elements of a vector operand.
2342 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2343 Value *OtherArg) {
2344 Constant *ShadowMul;
2345 Type *Ty = ConstArg->getType();
2346 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
2347 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2348 Type *EltTy = VTy->getElementType();
2349 SmallVector<Constant *, 16> Elements;
2350 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2351 if (ConstantInt *Elt =
2352 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2353 const APInt &V = Elt->getValue();
2354 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2355 Elements.push_back(ConstantInt::get(EltTy, V2));
2356 } else {
2357 Elements.push_back(ConstantInt::get(EltTy, 1));
2358 }
2359 }
2360 ShadowMul = ConstantVector::get(Elements);
2361 } else {
2362 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2363 const APInt &V = Elt->getValue();
2364 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2365 ShadowMul = ConstantInt::get(Ty, V2);
2366 } else {
2367 ShadowMul = ConstantInt::get(Ty, 1);
2368 }
2369 }
2370
2371 IRBuilder<> IRB(&I);
2372 setShadow(&I,
2373 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2374 setOrigin(&I, getOrigin(OtherArg));
2375 }
2376
2377 void visitMul(BinaryOperator &I) {
2378 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2379 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2380 if (constOp0 && !constOp1)
2381 handleMulByConstant(I, constOp0, I.getOperand(1));
2382 else if (constOp1 && !constOp0)
2383 handleMulByConstant(I, constOp1, I.getOperand(0));
2384 else
2385 handleShadowOr(I);
2386 }
2387
2388 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2389 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2390 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2391 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2392 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2393 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2394
2395 void handleIntegerDiv(Instruction &I) {
2396 IRBuilder<> IRB(&I);
2397 // Strict on the second argument.
2398 insertShadowCheck(I.getOperand(1), &I);
2399 setShadow(&I, getShadow(&I, 0));
2400 setOrigin(&I, getOrigin(&I, 0));
2401 }
2402
2403 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2404 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2405 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2406 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2407
2408 // Floating point division is side-effect free. We can not require that the
2409 // divisor is fully initialized and must propagate shadow. See PR37523.
2410 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2411 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2412
2413 /// Instrument == and != comparisons.
2414 ///
2415 /// Sometimes the comparison result is known even if some of the bits of the
2416 /// arguments are not.
2417 void handleEqualityComparison(ICmpInst &I) {
2418 IRBuilder<> IRB(&I);
2419 Value *A = I.getOperand(0);
2420 Value *B = I.getOperand(1);
2421 Value *Sa = getShadow(A);
2422 Value *Sb = getShadow(B);
2423
2424 // Get rid of pointers and vectors of pointers.
2425 // For ints (and vectors of ints), types of A and Sa match,
2426 // and this is a no-op.
2427 A = IRB.CreatePointerCast(A, Sa->getType());
2428 B = IRB.CreatePointerCast(B, Sb->getType());
2429
2430 // A == B <==> (C = A^B) == 0
2431 // A != B <==> (C = A^B) != 0
2432 // Sc = Sa | Sb
2433 Value *C = IRB.CreateXor(A, B);
2434 Value *Sc = IRB.CreateOr(Sa, Sb);
2435 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2436 // Result is defined if one of the following is true
2437 // * there is a defined 1 bit in C
2438 // * C is fully defined
2439 // Si = !(C & ~Sc) && Sc
2440 Value *Zero = Constant::getNullValue(Sc->getType());
2441 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2442 Value *Si = IRB.CreateAnd(
2443 IRB.CreateICmpNE(Sc, Zero),
2444 IRB.CreateICmpEQ(IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2445 Si->setName("_msprop_icmp");
2446 setShadow(&I, Si);
2447 setOriginForNaryOp(I);
2448 }
2449
2450 /// Build the lowest possible value of V, taking into account V's
2451 /// uninitialized bits.
2452 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2453 bool isSigned) {
2454 if (isSigned) {
2455 // Split shadow into sign bit and other bits.
2456 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2457 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2458 // Maximise the undefined shadow bit, minimize other undefined bits.
2459 return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)),
2460 SaSignBit);
2461 } else {
2462 // Minimize undefined bits.
2463 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2464 }
2465 }
2466
2467 /// Build the highest possible value of V, taking into account V's
2468 /// uninitialized bits.
2469 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2470 bool isSigned) {
2471 if (isSigned) {
2472 // Split shadow into sign bit and other bits.
2473 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2474 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2475 // Minimise the undefined shadow bit, maximise other undefined bits.
2476 return IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)),
2477 SaOtherBits);
2478 } else {
2479 // Maximize undefined bits.
2480 return IRB.CreateOr(A, Sa);
2481 }
2482 }
2483
2484 /// Instrument relational comparisons.
2485 ///
2486 /// This function does exact shadow propagation for all relational
2487 /// comparisons of integers, pointers and vectors of those.
2488 /// FIXME: output seems suboptimal when one of the operands is a constant
2489 void handleRelationalComparisonExact(ICmpInst &I) {
2490 IRBuilder<> IRB(&I);
2491 Value *A = I.getOperand(0);
2492 Value *B = I.getOperand(1);
2493 Value *Sa = getShadow(A);
2494 Value *Sb = getShadow(B);
2495
2496 // Get rid of pointers and vectors of pointers.
2497 // For ints (and vectors of ints), types of A and Sa match,
2498 // and this is a no-op.
2499 A = IRB.CreatePointerCast(A, Sa->getType());
2500 B = IRB.CreatePointerCast(B, Sb->getType());
2501
2502 // Let [a0, a1] be the interval of possible values of A, taking into account
2503 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2504 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2505 bool IsSigned = I.isSigned();
2506 Value *S1 = IRB.CreateICmp(I.getPredicate(),
2507 getLowestPossibleValue(IRB, A, Sa, IsSigned),
2508 getHighestPossibleValue(IRB, B, Sb, IsSigned));
2509 Value *S2 = IRB.CreateICmp(I.getPredicate(),
2510 getHighestPossibleValue(IRB, A, Sa, IsSigned),
2511 getLowestPossibleValue(IRB, B, Sb, IsSigned));
2512 Value *Si = IRB.CreateXor(S1, S2);
2513 setShadow(&I, Si);
2514 setOriginForNaryOp(I);
2515 }
2516
2517 /// Instrument signed relational comparisons.
2518 ///
2519 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2520 /// bit of the shadow. Everything else is delegated to handleShadowOr().
2521 void handleSignedRelationalComparison(ICmpInst &I) {
2522 Constant *constOp;
2523 Value *op = nullptr;
2524 CmpInst::Predicate pre;
2525 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2526 op = I.getOperand(0);
2527 pre = I.getPredicate();
2528 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2529 op = I.getOperand(1);
2530 pre = I.getSwappedPredicate();
2531 } else {
2532 handleShadowOr(I);
2533 return;
2534 }
2535
2536 if ((constOp->isNullValue() &&
2537 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2538 (constOp->isAllOnesValue() &&
2539 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2540 IRBuilder<> IRB(&I);
2541 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2542 "_msprop_icmp_s");
2543 setShadow(&I, Shadow);
2544 setOrigin(&I, getOrigin(op));
2545 } else {
2546 handleShadowOr(I);
2547 }
2548 }
2549
2550 void visitICmpInst(ICmpInst &I) {
2551 if (!ClHandleICmp) {
2552 handleShadowOr(I);
2553 return;
2554 }
2555 if (I.isEquality()) {
2556 handleEqualityComparison(I);
2557 return;
2558 }
2559
2560 assert(I.isRelational())(static_cast <bool> (I.isRelational()) ? void (0) : __assert_fail
("I.isRelational()", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2560, __extension__ __PRETTY_FUNCTION__))
;
2561 if (ClHandleICmpExact) {
2562 handleRelationalComparisonExact(I);
2563 return;
2564 }
2565 if (I.isSigned()) {
2566 handleSignedRelationalComparison(I);
2567 return;
2568 }
2569
2570 assert(I.isUnsigned())(static_cast <bool> (I.isUnsigned()) ? void (0) : __assert_fail
("I.isUnsigned()", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2570, __extension__ __PRETTY_FUNCTION__))
;
2571 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2572 handleRelationalComparisonExact(I);
2573 return;
2574 }
2575
2576 handleShadowOr(I);
2577 }
2578
2579 void visitFCmpInst(FCmpInst &I) { handleShadowOr(I); }
2580
2581 void handleShift(BinaryOperator &I) {
2582 IRBuilder<> IRB(&I);
2583 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2584 // Otherwise perform the same shift on S1.
2585 Value *S1 = getShadow(&I, 0);
2586 Value *S2 = getShadow(&I, 1);
2587 Value *S2Conv =
2588 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2589 Value *V2 = I.getOperand(1);
2590 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2591 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2592 setOriginForNaryOp(I);
2593 }
2594
2595 void visitShl(BinaryOperator &I) { handleShift(I); }
2596 void visitAShr(BinaryOperator &I) { handleShift(I); }
2597 void visitLShr(BinaryOperator &I) { handleShift(I); }
2598
2599 void handleFunnelShift(IntrinsicInst &I) {
2600 IRBuilder<> IRB(&I);
2601 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2602 // Otherwise perform the same shift on S0 and S1.
2603 Value *S0 = getShadow(&I, 0);
2604 Value *S1 = getShadow(&I, 1);
2605 Value *S2 = getShadow(&I, 2);
2606 Value *S2Conv =
2607 IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2608 Value *V2 = I.getOperand(2);
2609 Function *Intrin = Intrinsic::getDeclaration(
2610 I.getModule(), I.getIntrinsicID(), S2Conv->getType());
2611 Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
2612 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2613 setOriginForNaryOp(I);
2614 }
2615
2616 /// Instrument llvm.memmove
2617 ///
2618 /// At this point we don't know if llvm.memmove will be inlined or not.
2619 /// If we don't instrument it and it gets inlined,
2620 /// our interceptor will not kick in and we will lose the memmove.
2621 /// If we instrument the call here, but it does not get inlined,
2622 /// we will memove the shadow twice: which is bad in case
2623 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2624 ///
2625 /// Similar situation exists for memcpy and memset.
2626 void visitMemMoveInst(MemMoveInst &I) {
2627 getShadow(I.getArgOperand(1)); // Ensure shadow initialized
2628 IRBuilder<> IRB(&I);
2629 IRB.CreateCall(
2630 MS.MemmoveFn,
2631 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2632 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2633 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2634 I.eraseFromParent();
2635 }
2636
2637 /// Instrument memcpy
2638 ///
2639 /// Similar to memmove: avoid copying shadow twice. This is somewhat
2640 /// unfortunate as it may slowdown small constant memcpys.
2641 /// FIXME: consider doing manual inline for small constant sizes and proper
2642 /// alignment.
2643 ///
2644 /// Note: This also handles memcpy.inline, which promises no calls to external
2645 /// functions as an optimization. However, with instrumentation enabled this
2646 /// is difficult to promise; additionally, we know that the MSan runtime
2647 /// exists and provides __msan_memcpy(). Therefore, we assume that with
2648 /// instrumentation it's safe to turn memcpy.inline into a call to
2649 /// __msan_memcpy(). Should this be wrong, such as when implementing memcpy()
2650 /// itself, instrumentation should be disabled with the no_sanitize attribute.
2651 void visitMemCpyInst(MemCpyInst &I) {
2652 getShadow(I.getArgOperand(1)); // Ensure shadow initialized
2653 IRBuilder<> IRB(&I);
2654 IRB.CreateCall(
2655 MS.MemcpyFn,
2656 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2657 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2658 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2659 I.eraseFromParent();
2660 }
2661
2662 // Same as memcpy.
2663 void visitMemSetInst(MemSetInst &I) {
2664 IRBuilder<> IRB(&I);
2665 IRB.CreateCall(
2666 MS.MemsetFn,
2667 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2668 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2669 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2670 I.eraseFromParent();
2671 }
2672
2673 void visitVAStartInst(VAStartInst &I) { VAHelper->visitVAStartInst(I); }
2674
2675 void visitVACopyInst(VACopyInst &I) { VAHelper->visitVACopyInst(I); }
2676
2677 /// Handle vector store-like intrinsics.
2678 ///
2679 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2680 /// has 1 pointer argument and 1 vector argument, returns void.
2681 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2682 IRBuilder<> IRB(&I);
2683 Value *Addr = I.getArgOperand(0);
2684 Value *Shadow = getShadow(&I, 1);
2685 Value *ShadowPtr, *OriginPtr;
2686
2687 // We don't know the pointer alignment (could be unaligned SSE store!).
2688 // Have to assume to worst case.
2689 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2690 Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2691 IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
2692
2693 if (ClCheckAccessAddress)
2694 insertShadowCheck(Addr, &I);
2695
2696 // FIXME: factor out common code from materializeStores
2697 if (MS.TrackOrigins)
2698 IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2699 return true;
2700 }
2701
2702 /// Handle vector load-like intrinsics.
2703 ///
2704 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2705 /// has 1 pointer argument, returns a vector.
2706 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2707 IRBuilder<> IRB(&I);
2708 Value *Addr = I.getArgOperand(0);
2709
2710 Type *ShadowTy = getShadowTy(&I);
2711 Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2712 if (PropagateShadow) {
2713 // We don't know the pointer alignment (could be unaligned SSE load!).
2714 // Have to assume to worst case.
2715 const Align Alignment = Align(1);
2716 std::tie(ShadowPtr, OriginPtr) =
2717 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2718 setShadow(&I,
2719 IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2720 } else {
2721 setShadow(&I, getCleanShadow(&I));
2722 }
2723
2724 if (ClCheckAccessAddress)
2725 insertShadowCheck(Addr, &I);
2726
2727 if (MS.TrackOrigins) {
2728 if (PropagateShadow)
2729 setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2730 else
2731 setOrigin(&I, getCleanOrigin());
2732 }
2733 return true;
2734 }
2735
2736 /// Handle (SIMD arithmetic)-like intrinsics.
2737 ///
2738 /// Instrument intrinsics with any number of arguments of the same type,
2739 /// equal to the return type. The type should be simple (no aggregates or
2740 /// pointers; vectors are fine).
2741 /// Caller guarantees that this intrinsic does not access memory.
2742 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2743 Type *RetTy = I.getType();
2744 if (!(RetTy->isIntOrIntVectorTy() || RetTy->isFPOrFPVectorTy() ||
2745 RetTy->isX86_MMXTy()))
2746 return false;
2747
2748 unsigned NumArgOperands = I.arg_size();
2749 for (unsigned i = 0; i < NumArgOperands; ++i) {
2750 Type *Ty = I.getArgOperand(i)->getType();
2751 if (Ty != RetTy)
2752 return false;
2753 }
2754
2755 IRBuilder<> IRB(&I);
2756 ShadowAndOriginCombiner SC(this, IRB);
2757 for (unsigned i = 0; i < NumArgOperands; ++i)
2758 SC.Add(I.getArgOperand(i));
2759 SC.Done(&I);
2760
2761 return true;
2762 }
2763
2764 /// Heuristically instrument unknown intrinsics.
2765 ///
2766 /// The main purpose of this code is to do something reasonable with all
2767 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2768 /// We recognize several classes of intrinsics by their argument types and
2769 /// ModRefBehaviour and apply special instrumentation when we are reasonably
2770 /// sure that we know what the intrinsic does.
2771 ///
2772 /// We special-case intrinsics where this approach fails. See llvm.bswap
2773 /// handling as an example of that.
2774 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2775 unsigned NumArgOperands = I.arg_size();
2776 if (NumArgOperands == 0)
2777 return false;
2778
2779 if (NumArgOperands == 2 && I.getArgOperand(0)->getType()->isPointerTy() &&
2780 I.getArgOperand(1)->getType()->isVectorTy() &&
2781 I.getType()->isVoidTy() && !I.onlyReadsMemory()) {
2782 // This looks like a vector store.
2783 return handleVectorStoreIntrinsic(I);
2784 }
2785
2786 if (NumArgOperands == 1 && I.getArgOperand(0)->getType()->isPointerTy() &&
2787 I.getType()->isVectorTy() && I.onlyReadsMemory()) {
2788 // This looks like a vector load.
2789 return handleVectorLoadIntrinsic(I);
2790 }
2791
2792 if (I.doesNotAccessMemory())
2793 if (maybeHandleSimpleNomemIntrinsic(I))
2794 return true;
2795
2796 // FIXME: detect and handle SSE maskstore/maskload
2797 return false;
2798 }
2799
2800 void handleInvariantGroup(IntrinsicInst &I) {
2801 setShadow(&I, getShadow(&I, 0));
2802 setOrigin(&I, getOrigin(&I, 0));
2803 }
2804
2805 void handleLifetimeStart(IntrinsicInst &I) {
2806 if (!PoisonStack)
2807 return;
2808 AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
2809 if (!AI)
2810 InstrumentLifetimeStart = false;
2811 LifetimeStartList.push_back(std::make_pair(&I, AI));
2812 }
2813
2814 void handleBswap(IntrinsicInst &I) {
2815 IRBuilder<> IRB(&I);
2816 Value *Op = I.getArgOperand(0);
2817 Type *OpType = Op->getType();
2818 Function *BswapFunc = Intrinsic::getDeclaration(
2819 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2820 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2821 setOrigin(&I, getOrigin(Op));
2822 }
2823
2824 // Instrument vector convert intrinsic.
2825 //
2826 // This function instruments intrinsics like cvtsi2ss:
2827 // %Out = int_xxx_cvtyyy(%ConvertOp)
2828 // or
2829 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2830 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2831 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2832 // elements from \p CopyOp.
2833 // In most cases conversion involves floating-point value which may trigger a
2834 // hardware exception when not fully initialized. For this reason we require
2835 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2836 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2837 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2838 // return a fully initialized value.
2839 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
2840 bool HasRoundingMode = false) {
2841 IRBuilder<> IRB(&I);
2842 Value *CopyOp, *ConvertOp;
2843
2844 assert((!HasRoundingMode ||(static_cast <bool> ((!HasRoundingMode || isa<ConstantInt
>(I.getArgOperand(I.arg_size() - 1))) && "Invalid rounding mode"
) ? void (0) : __assert_fail ("(!HasRoundingMode || isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) && \"Invalid rounding mode\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2846
, __extension__ __PRETTY_FUNCTION__))
2845 isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) &&(static_cast <bool> ((!HasRoundingMode || isa<ConstantInt
>(I.getArgOperand(I.arg_size() - 1))) && "Invalid rounding mode"
) ? void (0) : __assert_fail ("(!HasRoundingMode || isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) && \"Invalid rounding mode\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2846
, __extension__ __PRETTY_FUNCTION__))
2846 "Invalid rounding mode")(static_cast <bool> ((!HasRoundingMode || isa<ConstantInt
>(I.getArgOperand(I.arg_size() - 1))) && "Invalid rounding mode"
) ? void (0) : __assert_fail ("(!HasRoundingMode || isa<ConstantInt>(I.getArgOperand(I.arg_size() - 1))) && \"Invalid rounding mode\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2846
, __extension__ __PRETTY_FUNCTION__))
;
2847
2848 switch (I.arg_size() - HasRoundingMode) {
2849 case 2:
2850 CopyOp = I.getArgOperand(0);
2851 ConvertOp = I.getArgOperand(1);
2852 break;
2853 case 1:
2854 ConvertOp = I.getArgOperand(0);
2855 CopyOp = nullptr;
2856 break;
2857 default:
2858 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.")::llvm::llvm_unreachable_internal("Cvt intrinsic with unsupported number of arguments."
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2858
)
;
2859 }
2860
2861 // The first *NumUsedElements* elements of ConvertOp are converted to the
2862 // same number of output elements. The rest of the output is copied from
2863 // CopyOp, or (if not available) filled with zeroes.
2864 // Combine shadow for elements of ConvertOp that are used in this operation,
2865 // and insert a check.
2866 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2867 // int->any conversion.
2868 Value *ConvertShadow = getShadow(ConvertOp);
2869 Value *AggShadow = nullptr;
2870 if (ConvertOp->getType()->isVectorTy()) {
2871 AggShadow = IRB.CreateExtractElement(
2872 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2873 for (int i = 1; i < NumUsedElements; ++i) {
2874 Value *MoreShadow = IRB.CreateExtractElement(
2875 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2876 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2877 }
2878 } else {
2879 AggShadow = ConvertShadow;
2880 }
2881 assert(AggShadow->getType()->isIntegerTy())(static_cast <bool> (AggShadow->getType()->isIntegerTy
()) ? void (0) : __assert_fail ("AggShadow->getType()->isIntegerTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2881
, __extension__ __PRETTY_FUNCTION__))
;
2882 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2883
2884 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2885 // ConvertOp.
2886 if (CopyOp) {
2887 assert(CopyOp->getType() == I.getType())(static_cast <bool> (CopyOp->getType() == I.getType(
)) ? void (0) : __assert_fail ("CopyOp->getType() == I.getType()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2887
, __extension__ __PRETTY_FUNCTION__))
;
2888 assert(CopyOp->getType()->isVectorTy())(static_cast <bool> (CopyOp->getType()->isVectorTy
()) ? void (0) : __assert_fail ("CopyOp->getType()->isVectorTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2888
, __extension__ __PRETTY_FUNCTION__))
;
2889 Value *ResultShadow = getShadow(CopyOp);
2890 Type *EltTy = cast<VectorType>(ResultShadow->getType())->getElementType();
2891 for (int i = 0; i < NumUsedElements; ++i) {
2892 ResultShadow = IRB.CreateInsertElement(
2893 ResultShadow, ConstantInt::getNullValue(EltTy),
2894 ConstantInt::get(IRB.getInt32Ty(), i));
2895 }
2896 setShadow(&I, ResultShadow);
2897 setOrigin(&I, getOrigin(CopyOp));
2898 } else {
2899 setShadow(&I, getCleanShadow(&I));
2900 setOrigin(&I, getCleanOrigin());
2901 }
2902 }
2903
2904 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2905 // zeroes if it is zero, and all ones otherwise.
2906 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2907 if (S->getType()->isVectorTy())
2908 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2909 assert(S->getType()->getPrimitiveSizeInBits() <= 64)(static_cast <bool> (S->getType()->getPrimitiveSizeInBits
() <= 64) ? void (0) : __assert_fail ("S->getType()->getPrimitiveSizeInBits() <= 64"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2909
, __extension__ __PRETTY_FUNCTION__))
;
2910 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2911 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2912 }
2913
2914 // Given a vector, extract its first element, and return all
2915 // zeroes if it is zero, and all ones otherwise.
2916 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2917 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2918 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2919 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2920 }
2921
2922 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2923 Type *T = S->getType();
2924 assert(T->isVectorTy())(static_cast <bool> (T->isVectorTy()) ? void (0) : __assert_fail
("T->isVectorTy()", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2924, __extension__ __PRETTY_FUNCTION__))
;
2925 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2926 return IRB.CreateSExt(S2, T);
2927 }
2928
2929 // Instrument vector shift intrinsic.
2930 //
2931 // This function instruments intrinsics like int_x86_avx2_psll_w.
2932 // Intrinsic shifts %In by %ShiftSize bits.
2933 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2934 // size, and the rest is ignored. Behavior is defined even if shift size is
2935 // greater than register (or field) width.
2936 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2937 assert(I.arg_size() == 2)(static_cast <bool> (I.arg_size() == 2) ? void (0) : __assert_fail
("I.arg_size() == 2", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2937, __extension__ __PRETTY_FUNCTION__))
;
2938 IRBuilder<> IRB(&I);
2939 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2940 // Otherwise perform the same shift on S1.
2941 Value *S1 = getShadow(&I, 0);
2942 Value *S2 = getShadow(&I, 1);
2943 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2944 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2945 Value *V1 = I.getOperand(0);
2946 Value *V2 = I.getOperand(1);
2947 Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
2948 {IRB.CreateBitCast(S1, V1->getType()), V2});
2949 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2950 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2951 setOriginForNaryOp(I);
2952 }
2953
2954 // Get an X86_MMX-sized vector type.
2955 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2956 const unsigned X86_MMXSizeInBits = 64;
2957 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&(static_cast <bool> (EltSizeInBits != 0 && (X86_MMXSizeInBits
% EltSizeInBits) == 0 && "Illegal MMX vector element size"
) ? void (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2958
, __extension__ __PRETTY_FUNCTION__))
2958 "Illegal MMX vector element size")(static_cast <bool> (EltSizeInBits != 0 && (X86_MMXSizeInBits
% EltSizeInBits) == 0 && "Illegal MMX vector element size"
) ? void (0) : __assert_fail ("EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 && \"Illegal MMX vector element size\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 2958
, __extension__ __PRETTY_FUNCTION__))
;
2959 return FixedVectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2960 X86_MMXSizeInBits / EltSizeInBits);
2961 }
2962
2963 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2964 // intrinsic.
2965 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2966 switch (id) {
2967 case Intrinsic::x86_sse2_packsswb_128:
2968 case Intrinsic::x86_sse2_packuswb_128:
2969 return Intrinsic::x86_sse2_packsswb_128;
2970
2971 case Intrinsic::x86_sse2_packssdw_128:
2972 case Intrinsic::x86_sse41_packusdw:
2973 return Intrinsic::x86_sse2_packssdw_128;
2974
2975 case Intrinsic::x86_avx2_packsswb:
2976 case Intrinsic::x86_avx2_packuswb:
2977 return Intrinsic::x86_avx2_packsswb;
2978
2979 case Intrinsic::x86_avx2_packssdw:
2980 case Intrinsic::x86_avx2_packusdw:
2981 return Intrinsic::x86_avx2_packssdw;
2982
2983 case Intrinsic::x86_mmx_packsswb:
2984 case Intrinsic::x86_mmx_packuswb:
2985 return Intrinsic::x86_mmx_packsswb;
2986
2987 case Intrinsic::x86_mmx_packssdw:
2988 return Intrinsic::x86_mmx_packssdw;
2989 default:
2990 llvm_unreachable("unexpected intrinsic id")::llvm::llvm_unreachable_internal("unexpected intrinsic id", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 2990)
;
2991 }
2992 }
2993
2994 // Instrument vector pack intrinsic.
2995 //
2996 // This function instruments intrinsics like x86_mmx_packsswb, that
2997 // packs elements of 2 input vectors into half as many bits with saturation.
2998 // Shadow is propagated with the signed variant of the same intrinsic applied
2999 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
3000 // EltSizeInBits is used only for x86mmx arguments.
3001 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
3002 assert(I.arg_size() == 2)(static_cast <bool> (I.arg_size() == 2) ? void (0) : __assert_fail
("I.arg_size() == 2", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3002, __extension__ __PRETTY_FUNCTION__))
;
3003 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
3004 IRBuilder<> IRB(&I);
3005 Value *S1 = getShadow(&I, 0);
3006 Value *S2 = getShadow(&I, 1);
3007 assert(isX86_MMX || S1->getType()->isVectorTy())(static_cast <bool> (isX86_MMX || S1->getType()->
isVectorTy()) ? void (0) : __assert_fail ("isX86_MMX || S1->getType()->isVectorTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3007
, __extension__ __PRETTY_FUNCTION__))
;
3008
3009 // SExt and ICmpNE below must apply to individual elements of input vectors.
3010 // In case of x86mmx arguments, cast them to appropriate vector types and
3011 // back.
3012 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
3013 if (isX86_MMX) {
3014 S1 = IRB.CreateBitCast(S1, T);
3015 S2 = IRB.CreateBitCast(S2, T);
3016 }
3017 Value *S1_ext =
3018 IRB.CreateSExt(IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
3019 Value *S2_ext =
3020 IRB.CreateSExt(IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
3021 if (isX86_MMX) {
3022 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
3023 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
3024 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
3025 }
3026
3027 Function *ShadowFn = Intrinsic::getDeclaration(
3028 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
3029
3030 Value *S =
3031 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
3032 if (isX86_MMX)
3033 S = IRB.CreateBitCast(S, getShadowTy(&I));
3034 setShadow(&I, S);
3035 setOriginForNaryOp(I);
3036 }
3037
3038 // Instrument sum-of-absolute-differences intrinsic.
3039 void handleVectorSadIntrinsic(IntrinsicInst &I) {
3040 const unsigned SignificantBitsPerResultElement = 16;
3041 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
3042 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
3043 unsigned ZeroBitsPerResultElement =
3044 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
3045
3046 IRBuilder<> IRB(&I);
3047 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3048 S = IRB.CreateBitCast(S, ResTy);
3049 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
3050 ResTy);
3051 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
3052 S = IRB.CreateBitCast(S, getShadowTy(&I));
3053 setShadow(&I, S);
3054 setOriginForNaryOp(I);
3055 }
3056
3057 // Instrument multiply-add intrinsic.
3058 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
3059 unsigned EltSizeInBits = 0) {
3060 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
3061 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
3062 IRBuilder<> IRB(&I);
3063 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3064 S = IRB.CreateBitCast(S, ResTy);
3065 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
3066 ResTy);
3067 S = IRB.CreateBitCast(S, getShadowTy(&I));
3068 setShadow(&I, S);
3069 setOriginForNaryOp(I);
3070 }
3071
3072 // Instrument compare-packed intrinsic.
3073 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
3074 // all-ones shadow.
3075 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
3076 IRBuilder<> IRB(&I);
3077 Type *ResTy = getShadowTy(&I);
3078 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3079 Value *S = IRB.CreateSExt(
3080 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
3081 setShadow(&I, S);
3082 setOriginForNaryOp(I);
3083 }
3084
3085 // Instrument compare-scalar intrinsic.
3086 // This handles both cmp* intrinsics which return the result in the first
3087 // element of a vector, and comi* which return the result as i32.
3088 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
3089 IRBuilder<> IRB(&I);
3090 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3091 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
3092 setShadow(&I, S);
3093 setOriginForNaryOp(I);
3094 }
3095
3096 // Instrument generic vector reduction intrinsics
3097 // by ORing together all their fields.
3098 void handleVectorReduceIntrinsic(IntrinsicInst &I) {
3099 IRBuilder<> IRB(&I);
3100 Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
3101 setShadow(&I, S);
3102 setOrigin(&I, getOrigin(&I, 0));
3103 }
3104
3105 // Instrument vector.reduce.or intrinsic.
3106 // Valid (non-poisoned) set bits in the operand pull low the
3107 // corresponding shadow bits.
3108 void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
3109 IRBuilder<> IRB(&I);
3110 Value *OperandShadow = getShadow(&I, 0);
3111 Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3112 Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3113 // Bit N is clean if any field's bit N is 1 and unpoison
3114 Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3115 // Otherwise, it is clean if every field's bit N is unpoison
3116 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3117 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3118
3119 setShadow(&I, S);
3120 setOrigin(&I, getOrigin(&I, 0));
3121 }
3122
3123 // Instrument vector.reduce.and intrinsic.
3124 // Valid (non-poisoned) unset bits in the operand pull down the
3125 // corresponding shadow bits.
3126 void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
3127 IRBuilder<> IRB(&I);
3128 Value *OperandShadow = getShadow(&I, 0);
3129 Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3130 // Bit N is clean if any field's bit N is 0 and unpoison
3131 Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3132 // Otherwise, it is clean if every field's bit N is unpoison
3133 Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3134 Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3135
3136 setShadow(&I, S);
3137 setOrigin(&I, getOrigin(&I, 0));
3138 }
3139
3140 void handleStmxcsr(IntrinsicInst &I) {
3141 IRBuilder<> IRB(&I);
3142 Value *Addr = I.getArgOperand(0);
3143 Type *Ty = IRB.getInt32Ty();
3144 Value *ShadowPtr =
3145 getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
3146
3147 IRB.CreateStore(getCleanShadow(Ty),
3148 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
3149
3150 if (ClCheckAccessAddress)
3151 insertShadowCheck(Addr, &I);
3152 }
3153
3154 void handleLdmxcsr(IntrinsicInst &I) {
3155 if (!InsertChecks)
3156 return;
3157
3158 IRBuilder<> IRB(&I);
3159 Value *Addr = I.getArgOperand(0);
3160 Type *Ty = IRB.getInt32Ty();
3161 const Align Alignment = Align(1);
3162 Value *ShadowPtr, *OriginPtr;
3163 std::tie(ShadowPtr, OriginPtr) =
3164 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
3165
3166 if (ClCheckAccessAddress)
3167 insertShadowCheck(Addr, &I);
3168
3169 Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
3170 Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
3171 : getCleanOrigin();
3172 insertShadowCheck(Shadow, Origin, &I);
3173 }
3174
3175 void handleMaskedStore(IntrinsicInst &I) {
3176 IRBuilder<> IRB(&I);
3177 Value *V = I.getArgOperand(0);
3178 Value *Addr = I.getArgOperand(1);
3179 const Align Alignment(
3180 cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
3181 Value *Mask = I.getArgOperand(3);
3182 Value *Shadow = getShadow(V);
3183
3184 Value *ShadowPtr;
3185 Value *OriginPtr;
3186 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3187 Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3188
3189 if (ClCheckAccessAddress) {
3190 insertShadowCheck(Addr, &I);
3191 // Uninitialized mask is kind of like uninitialized address, but not as
3192 // scary.
3193 insertShadowCheck(Mask, &I);
3194 }
3195
3196 IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3197
3198 if (MS.TrackOrigins) {
3199 auto &DL = F.getParent()->getDataLayout();
3200 paintOrigin(IRB, getOrigin(V), OriginPtr,
3201 DL.getTypeStoreSize(Shadow->getType()),
3202 std::max(Alignment, kMinOriginAlignment));
3203 }
3204 }
3205
3206 bool handleMaskedLoad(IntrinsicInst &I) {
3207 IRBuilder<> IRB(&I);
3208 Value *Addr = I.getArgOperand(0);
3209 const Align Alignment(
3210 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
3211 Value *Mask = I.getArgOperand(2);
3212 Value *PassThru = I.getArgOperand(3);
3213
3214 Type *ShadowTy = getShadowTy(&I);
3215 Value *ShadowPtr, *OriginPtr;
3216 if (PropagateShadow) {
3217 std::tie(ShadowPtr, OriginPtr) =
3218 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3219 setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3220 getShadow(PassThru), "_msmaskedld"));
3221 } else {
3222 setShadow(&I, getCleanShadow(&I));
3223 }
3224
3225 if (ClCheckAccessAddress) {
3226 insertShadowCheck(Addr, &I);
3227 insertShadowCheck(Mask, &I);
3228 }
3229
3230 if (MS.TrackOrigins) {
3231 if (PropagateShadow) {
3232 // Choose between PassThru's and the loaded value's origins.
3233 Value *MaskedPassThruShadow = IRB.CreateAnd(
3234 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
3235
3236 Value *Acc = IRB.CreateExtractElement(
3237 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
3238 for (int i = 1, N = cast<FixedVectorType>(PassThru->getType())
3239 ->getNumElements();
3240 i < N; ++i) {
3241 Value *More = IRB.CreateExtractElement(
3242 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
3243 Acc = IRB.CreateOr(Acc, More);
3244 }
3245
3246 Value *Origin = IRB.CreateSelect(
3247 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
3248 getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
3249
3250 setOrigin(&I, Origin);
3251 } else {
3252 setOrigin(&I, getCleanOrigin());
3253 }
3254 }
3255 return true;
3256 }
3257
3258 // Instrument BMI / BMI2 intrinsics.
3259 // All of these intrinsics are Z = I(X, Y)
3260 // where the types of all operands and the result match, and are either i32 or
3261 // i64. The following instrumentation happens to work for all of them:
3262 // Sz = I(Sx, Y) | (sext (Sy != 0))
3263 void handleBmiIntrinsic(IntrinsicInst &I) {
3264 IRBuilder<> IRB(&I);
3265 Type *ShadowTy = getShadowTy(&I);
3266
3267 // If any bit of the mask operand is poisoned, then the whole thing is.
3268 Value *SMask = getShadow(&I, 1);
3269 SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3270 ShadowTy);
3271 // Apply the same intrinsic to the shadow of the first operand.
3272 Value *S = IRB.CreateCall(I.getCalledFunction(),
3273 {getShadow(&I, 0), I.getOperand(1)});
3274 S = IRB.CreateOr(SMask, S);
3275 setShadow(&I, S);
3276 setOriginForNaryOp(I);
3277 }
3278
3279 SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
3280 SmallVector<int, 8> Mask;
3281 for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
3282 Mask.append(2, X);
3283 }
3284 return Mask;
3285 }
3286
3287 // Instrument pclmul intrinsics.
3288 // These intrinsics operate either on odd or on even elements of the input
3289 // vectors, depending on the constant in the 3rd argument, ignoring the rest.
3290 // Replace the unused elements with copies of the used ones, ex:
3291 // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
3292 // or
3293 // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
3294 // and then apply the usual shadow combining logic.
3295 void handlePclmulIntrinsic(IntrinsicInst &I) {
3296 IRBuilder<> IRB(&I);
3297 unsigned Width =
3298 cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3299 assert(isa<ConstantInt>(I.getArgOperand(2)) &&(static_cast <bool> (isa<ConstantInt>(I.getArgOperand
(2)) && "pclmul 3rd operand must be a constant") ? void
(0) : __assert_fail ("isa<ConstantInt>(I.getArgOperand(2)) && \"pclmul 3rd operand must be a constant\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3300
, __extension__ __PRETTY_FUNCTION__))
3300 "pclmul 3rd operand must be a constant")(static_cast <bool> (isa<ConstantInt>(I.getArgOperand
(2)) && "pclmul 3rd operand must be a constant") ? void
(0) : __assert_fail ("isa<ConstantInt>(I.getArgOperand(2)) && \"pclmul 3rd operand must be a constant\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3300
, __extension__ __PRETTY_FUNCTION__))
;
3301 unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3302 Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3303 getPclmulMask(Width, Imm & 0x01));
3304 Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3305 getPclmulMask(Width, Imm & 0x10));
3306 ShadowAndOriginCombiner SOC(this, IRB);
3307 SOC.Add(Shuf0, getOrigin(&I, 0));
3308 SOC.Add(Shuf1, getOrigin(&I, 1));
3309 SOC.Done(&I);
3310 }
3311
3312 // Instrument _mm_*_sd|ss intrinsics
3313 void handleUnarySdSsIntrinsic(IntrinsicInst &I) {
3314 IRBuilder<> IRB(&I);
3315 unsigned Width =
3316 cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3317 Value *First = getShadow(&I, 0);
3318 Value *Second = getShadow(&I, 1);
3319 // First element of second operand, remaining elements of first operand
3320 SmallVector<int, 16> Mask;
3321 Mask.push_back(Width);
3322 for (unsigned i = 1; i < Width; i++)
3323 Mask.push_back(i);
3324 Value *Shadow = IRB.CreateShuffleVector(First, Second, Mask);
3325
3326 setShadow(&I, Shadow);
3327 setOriginForNaryOp(I);
3328 }
3329
3330 void handleBinarySdSsIntrinsic(IntrinsicInst &I) {
3331 IRBuilder<> IRB(&I);
3332 unsigned Width =
3333 cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3334 Value *First = getShadow(&I, 0);
3335 Value *Second = getShadow(&I, 1);
3336 Value *OrShadow = IRB.CreateOr(First, Second);
3337 // First element of both OR'd together, remaining elements of first operand
3338 SmallVector<int, 16> Mask;
3339 Mask.push_back(Width);
3340 for (unsigned i = 1; i < Width; i++)
3341 Mask.push_back(i);
3342 Value *Shadow = IRB.CreateShuffleVector(First, OrShadow, Mask);
3343
3344 setShadow(&I, Shadow);
3345 setOriginForNaryOp(I);
3346 }
3347
3348 // Instrument abs intrinsic.
3349 // handleUnknownIntrinsic can't handle it because of the last
3350 // is_int_min_poison argument which does not match the result type.
3351 void handleAbsIntrinsic(IntrinsicInst &I) {
3352 assert(I.getType()->isIntOrIntVectorTy())(static_cast <bool> (I.getType()->isIntOrIntVectorTy
()) ? void (0) : __assert_fail ("I.getType()->isIntOrIntVectorTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3352
, __extension__ __PRETTY_FUNCTION__))
;
3353 assert(I.getArgOperand(0)->getType() == I.getType())(static_cast <bool> (I.getArgOperand(0)->getType() ==
I.getType()) ? void (0) : __assert_fail ("I.getArgOperand(0)->getType() == I.getType()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3353
, __extension__ __PRETTY_FUNCTION__))
;
3354
3355 // FIXME: Handle is_int_min_poison.
3356 IRBuilder<> IRB(&I);
3357 setShadow(&I, getShadow(&I, 0));
3358 setOrigin(&I, getOrigin(&I, 0));
3359 }
3360
3361 void visitIntrinsicInst(IntrinsicInst &I) {
3362 switch (I.getIntrinsicID()) {
3363 case Intrinsic::abs:
3364 handleAbsIntrinsic(I);
3365 break;
3366 case Intrinsic::lifetime_start:
3367 handleLifetimeStart(I);
3368 break;
3369 case Intrinsic::launder_invariant_group:
3370 case Intrinsic::strip_invariant_group:
3371 handleInvariantGroup(I);
3372 break;
3373 case Intrinsic::bswap:
3374 handleBswap(I);
3375 break;
3376 case Intrinsic::masked_store:
3377 handleMaskedStore(I);
3378 break;
3379 case Intrinsic::masked_load:
3380 handleMaskedLoad(I);
3381 break;
3382 case Intrinsic::vector_reduce_and:
3383 handleVectorReduceAndIntrinsic(I);
3384 break;
3385 case Intrinsic::vector_reduce_or:
3386 handleVectorReduceOrIntrinsic(I);
3387 break;
3388 case Intrinsic::vector_reduce_add:
3389 case Intrinsic::vector_reduce_xor:
3390 case Intrinsic::vector_reduce_mul:
3391 handleVectorReduceIntrinsic(I);
3392 break;
3393 case Intrinsic::x86_sse_stmxcsr:
3394 handleStmxcsr(I);
3395 break;
3396 case Intrinsic::x86_sse_ldmxcsr:
3397 handleLdmxcsr(I);
3398 break;
3399 case Intrinsic::x86_avx512_vcvtsd2usi64:
3400 case Intrinsic::x86_avx512_vcvtsd2usi32:
3401 case Intrinsic::x86_avx512_vcvtss2usi64:
3402 case Intrinsic::x86_avx512_vcvtss2usi32:
3403 case Intrinsic::x86_avx512_cvttss2usi64:
3404 case Intrinsic::x86_avx512_cvttss2usi:
3405 case Intrinsic::x86_avx512_cvttsd2usi64:
3406 case Intrinsic::x86_avx512_cvttsd2usi:
3407 case Intrinsic::x86_avx512_cvtusi2ss:
3408 case Intrinsic::x86_avx512_cvtusi642sd:
3409 case Intrinsic::x86_avx512_cvtusi642ss:
3410 handleVectorConvertIntrinsic(I, 1, true);
3411 break;
3412 case Intrinsic::x86_sse2_cvtsd2si64:
3413 case Intrinsic::x86_sse2_cvtsd2si:
3414 case Intrinsic::x86_sse2_cvtsd2ss:
3415 case Intrinsic::x86_sse2_cvttsd2si64:
3416 case Intrinsic::x86_sse2_cvttsd2si:
3417 case Intrinsic::x86_sse_cvtss2si64:
3418 case Intrinsic::x86_sse_cvtss2si:
3419 case Intrinsic::x86_sse_cvttss2si64:
3420 case Intrinsic::x86_sse_cvttss2si:
3421 handleVectorConvertIntrinsic(I, 1);
3422 break;
3423 case Intrinsic::x86_sse_cvtps2pi:
3424 case Intrinsic::x86_sse_cvttps2pi:
3425 handleVectorConvertIntrinsic(I, 2);
3426 break;
3427
3428 case Intrinsic::x86_avx512_psll_w_512:
3429 case Intrinsic::x86_avx512_psll_d_512:
3430 case Intrinsic::x86_avx512_psll_q_512:
3431 case Intrinsic::x86_avx512_pslli_w_512:
3432 case Intrinsic::x86_avx512_pslli_d_512:
3433 case Intrinsic::x86_avx512_pslli_q_512:
3434 case Intrinsic::x86_avx512_psrl_w_512:
3435 case Intrinsic::x86_avx512_psrl_d_512:
3436 case Intrinsic::x86_avx512_psrl_q_512:
3437 case Intrinsic::x86_avx512_psra_w_512:
3438 case Intrinsic::x86_avx512_psra_d_512:
3439 case Intrinsic::x86_avx512_psra_q_512:
3440 case Intrinsic::x86_avx512_psrli_w_512:
3441 case Intrinsic::x86_avx512_psrli_d_512:
3442 case Intrinsic::x86_avx512_psrli_q_512:
3443 case Intrinsic::x86_avx512_psrai_w_512:
3444 case Intrinsic::x86_avx512_psrai_d_512:
3445 case Intrinsic::x86_avx512_psrai_q_512:
3446 case Intrinsic::x86_avx512_psra_q_256:
3447 case Intrinsic::x86_avx512_psra_q_128:
3448 case Intrinsic::x86_avx512_psrai_q_256:
3449 case Intrinsic::x86_avx512_psrai_q_128:
3450 case Intrinsic::x86_avx2_psll_w:
3451 case Intrinsic::x86_avx2_psll_d:
3452 case Intrinsic::x86_avx2_psll_q:
3453 case Intrinsic::x86_avx2_pslli_w:
3454 case Intrinsic::x86_avx2_pslli_d:
3455 case Intrinsic::x86_avx2_pslli_q:
3456 case Intrinsic::x86_avx2_psrl_w:
3457 case Intrinsic::x86_avx2_psrl_d:
3458 case Intrinsic::x86_avx2_psrl_q:
3459 case Intrinsic::x86_avx2_psra_w:
3460 case Intrinsic::x86_avx2_psra_d:
3461 case Intrinsic::x86_avx2_psrli_w:
3462 case Intrinsic::x86_avx2_psrli_d:
3463 case Intrinsic::x86_avx2_psrli_q:
3464 case Intrinsic::x86_avx2_psrai_w:
3465 case Intrinsic::x86_avx2_psrai_d:
3466 case Intrinsic::x86_sse2_psll_w:
3467 case Intrinsic::x86_sse2_psll_d:
3468 case Intrinsic::x86_sse2_psll_q:
3469 case Intrinsic::x86_sse2_pslli_w:
3470 case Intrinsic::x86_sse2_pslli_d:
3471 case Intrinsic::x86_sse2_pslli_q:
3472 case Intrinsic::x86_sse2_psrl_w:
3473 case Intrinsic::x86_sse2_psrl_d:
3474 case Intrinsic::x86_sse2_psrl_q:
3475 case Intrinsic::x86_sse2_psra_w:
3476 case Intrinsic::x86_sse2_psra_d:
3477 case Intrinsic::x86_sse2_psrli_w:
3478 case Intrinsic::x86_sse2_psrli_d:
3479 case Intrinsic::x86_sse2_psrli_q:
3480 case Intrinsic::x86_sse2_psrai_w:
3481 case Intrinsic::x86_sse2_psrai_d:
3482 case Intrinsic::x86_mmx_psll_w:
3483 case Intrinsic::x86_mmx_psll_d:
3484 case Intrinsic::x86_mmx_psll_q:
3485 case Intrinsic::x86_mmx_pslli_w:
3486 case Intrinsic::x86_mmx_pslli_d:
3487 case Intrinsic::x86_mmx_pslli_q:
3488 case Intrinsic::x86_mmx_psrl_w:
3489 case Intrinsic::x86_mmx_psrl_d:
3490 case Intrinsic::x86_mmx_psrl_q:
3491 case Intrinsic::x86_mmx_psra_w:
3492 case Intrinsic::x86_mmx_psra_d:
3493 case Intrinsic::x86_mmx_psrli_w:
3494 case Intrinsic::x86_mmx_psrli_d:
3495 case Intrinsic::x86_mmx_psrli_q:
3496 case Intrinsic::x86_mmx_psrai_w:
3497 case Intrinsic::x86_mmx_psrai_d:
3498 handleVectorShiftIntrinsic(I, /* Variable */ false);
3499 break;
3500 case Intrinsic::x86_avx2_psllv_d:
3501 case Intrinsic::x86_avx2_psllv_d_256:
3502 case Intrinsic::x86_avx512_psllv_d_512:
3503 case Intrinsic::x86_avx2_psllv_q:
3504 case Intrinsic::x86_avx2_psllv_q_256:
3505 case Intrinsic::x86_avx512_psllv_q_512:
3506 case Intrinsic::x86_avx2_psrlv_d:
3507 case Intrinsic::x86_avx2_psrlv_d_256:
3508 case Intrinsic::x86_avx512_psrlv_d_512:
3509 case Intrinsic::x86_avx2_psrlv_q:
3510 case Intrinsic::x86_avx2_psrlv_q_256:
3511 case Intrinsic::x86_avx512_psrlv_q_512:
3512 case Intrinsic::x86_avx2_psrav_d:
3513 case Intrinsic::x86_avx2_psrav_d_256:
3514 case Intrinsic::x86_avx512_psrav_d_512:
3515 case Intrinsic::x86_avx512_psrav_q_128:
3516 case Intrinsic::x86_avx512_psrav_q_256:
3517 case Intrinsic::x86_avx512_psrav_q_512:
3518 handleVectorShiftIntrinsic(I, /* Variable */ true);
3519 break;
3520
3521 case Intrinsic::x86_sse2_packsswb_128:
3522 case Intrinsic::x86_sse2_packssdw_128:
3523 case Intrinsic::x86_sse2_packuswb_128:
3524 case Intrinsic::x86_sse41_packusdw:
3525 case Intrinsic::x86_avx2_packsswb:
3526 case Intrinsic::x86_avx2_packssdw:
3527 case Intrinsic::x86_avx2_packuswb:
3528 case Intrinsic::x86_avx2_packusdw:
3529 handleVectorPackIntrinsic(I);
3530 break;
3531
3532 case Intrinsic::x86_mmx_packsswb:
3533 case Intrinsic::x86_mmx_packuswb:
3534 handleVectorPackIntrinsic(I, 16);
3535 break;
3536
3537 case Intrinsic::x86_mmx_packssdw:
3538 handleVectorPackIntrinsic(I, 32);
3539 break;
3540
3541 case Intrinsic::x86_mmx_psad_bw:
3542 case Intrinsic::x86_sse2_psad_bw:
3543 case Intrinsic::x86_avx2_psad_bw:
3544 handleVectorSadIntrinsic(I);
3545 break;
3546
3547 case Intrinsic::x86_sse2_pmadd_wd:
3548 case Intrinsic::x86_avx2_pmadd_wd:
3549 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3550 case Intrinsic::x86_avx2_pmadd_ub_sw:
3551 handleVectorPmaddIntrinsic(I);
3552 break;
3553
3554 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3555 handleVectorPmaddIntrinsic(I, 8);
3556 break;
3557
3558 case Intrinsic::x86_mmx_pmadd_wd:
3559 handleVectorPmaddIntrinsic(I, 16);
3560 break;
3561
3562 case Intrinsic::x86_sse_cmp_ss:
3563 case Intrinsic::x86_sse2_cmp_sd:
3564 case Intrinsic::x86_sse_comieq_ss:
3565 case Intrinsic::x86_sse_comilt_ss:
3566 case Intrinsic::x86_sse_comile_ss:
3567 case Intrinsic::x86_sse_comigt_ss:
3568 case Intrinsic::x86_sse_comige_ss:
3569 case Intrinsic::x86_sse_comineq_ss:
3570 case Intrinsic::x86_sse_ucomieq_ss:
3571 case Intrinsic::x86_sse_ucomilt_ss:
3572 case Intrinsic::x86_sse_ucomile_ss:
3573 case Intrinsic::x86_sse_ucomigt_ss:
3574 case Intrinsic::x86_sse_ucomige_ss:
3575 case Intrinsic::x86_sse_ucomineq_ss:
3576 case Intrinsic::x86_sse2_comieq_sd:
3577 case Intrinsic::x86_sse2_comilt_sd:
3578 case Intrinsic::x86_sse2_comile_sd:
3579 case Intrinsic::x86_sse2_comigt_sd:
3580 case Intrinsic::x86_sse2_comige_sd:
3581 case Intrinsic::x86_sse2_comineq_sd:
3582 case Intrinsic::x86_sse2_ucomieq_sd:
3583 case Intrinsic::x86_sse2_ucomilt_sd:
3584 case Intrinsic::x86_sse2_ucomile_sd:
3585 case Intrinsic::x86_sse2_ucomigt_sd:
3586 case Intrinsic::x86_sse2_ucomige_sd:
3587 case Intrinsic::x86_sse2_ucomineq_sd:
3588 handleVectorCompareScalarIntrinsic(I);
3589 break;
3590
3591 case Intrinsic::x86_sse_cmp_ps:
3592 case Intrinsic::x86_sse2_cmp_pd:
3593 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3594 // generates reasonably looking IR that fails in the backend with "Do not
3595 // know how to split the result of this operator!".
3596 handleVectorComparePackedIntrinsic(I);
3597 break;
3598
3599 case Intrinsic::x86_bmi_bextr_32:
3600 case Intrinsic::x86_bmi_bextr_64:
3601 case Intrinsic::x86_bmi_bzhi_32:
3602 case Intrinsic::x86_bmi_bzhi_64:
3603 case Intrinsic::x86_bmi_pdep_32:
3604 case Intrinsic::x86_bmi_pdep_64:
3605 case Intrinsic::x86_bmi_pext_32:
3606 case Intrinsic::x86_bmi_pext_64:
3607 handleBmiIntrinsic(I);
3608 break;
3609
3610 case Intrinsic::x86_pclmulqdq:
3611 case Intrinsic::x86_pclmulqdq_256:
3612 case Intrinsic::x86_pclmulqdq_512:
3613 handlePclmulIntrinsic(I);
3614 break;
3615
3616 case Intrinsic::x86_sse41_round_sd:
3617 case Intrinsic::x86_sse41_round_ss:
3618 handleUnarySdSsIntrinsic(I);
3619 break;
3620 case Intrinsic::x86_sse2_max_sd:
3621 case Intrinsic::x86_sse_max_ss:
3622 case Intrinsic::x86_sse2_min_sd:
3623 case Intrinsic::x86_sse_min_ss:
3624 handleBinarySdSsIntrinsic(I);
3625 break;
3626
3627 case Intrinsic::fshl:
3628 case Intrinsic::fshr:
3629 handleFunnelShift(I);
3630 break;
3631
3632 case Intrinsic::is_constant:
3633 // The result of llvm.is.constant() is always defined.
3634 setShadow(&I, getCleanShadow(&I));
3635 setOrigin(&I, getCleanOrigin());
3636 break;
3637
3638 default:
3639 if (!handleUnknownIntrinsic(I))
3640 visitInstruction(I);
3641 break;
3642 }
3643 }
3644
3645 void visitLibAtomicLoad(CallBase &CB) {
3646 // Since we use getNextNode here, we can't have CB terminate the BB.
3647 assert(isa<CallInst>(CB))(static_cast <bool> (isa<CallInst>(CB)) ? void (0
) : __assert_fail ("isa<CallInst>(CB)", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3647, __extension__ __PRETTY_FUNCTION__))
;
3648
3649 IRBuilder<> IRB(&CB);
3650 Value *Size = CB.getArgOperand(0);
3651 Value *SrcPtr = CB.getArgOperand(1);
3652 Value *DstPtr = CB.getArgOperand(2);
3653 Value *Ordering = CB.getArgOperand(3);
3654 // Convert the call to have at least Acquire ordering to make sure
3655 // the shadow operations aren't reordered before it.
3656 Value *NewOrdering =
3657 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3658 CB.setArgOperand(3, NewOrdering);
3659
3660 NextNodeIRBuilder NextIRB(&CB);
3661 Value *SrcShadowPtr, *SrcOriginPtr;
3662 std::tie(SrcShadowPtr, SrcOriginPtr) =
3663 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3664 /*isStore*/ false);
3665 Value *DstShadowPtr =
3666 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3667 /*isStore*/ true)
3668 .first;
3669
3670 NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
3671 if (MS.TrackOrigins) {
3672 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
3673 kMinOriginAlignment);
3674 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
3675 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
3676 }
3677 }
3678
3679 void visitLibAtomicStore(CallBase &CB) {
3680 IRBuilder<> IRB(&CB);
3681 Value *Size = CB.getArgOperand(0);
3682 Value *DstPtr = CB.getArgOperand(2);
3683 Value *Ordering = CB.getArgOperand(3);
3684 // Convert the call to have at least Release ordering to make sure
3685 // the shadow operations aren't reordered after it.
3686 Value *NewOrdering =
3687 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3688 CB.setArgOperand(3, NewOrdering);
3689
3690 Value *DstShadowPtr =
3691 getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
3692 /*isStore*/ true)
3693 .first;
3694
3695 // Atomic store always paints clean shadow/origin. See file header.
3696 IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
3697 Align(1));
3698 }
3699
3700 void visitCallBase(CallBase &CB) {
3701 assert(!CB.getMetadata(LLVMContext::MD_nosanitize))(static_cast <bool> (!CB.getMetadata(LLVMContext::MD_nosanitize
)) ? void (0) : __assert_fail ("!CB.getMetadata(LLVMContext::MD_nosanitize)"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3701
, __extension__ __PRETTY_FUNCTION__))
;
3702 if (CB.isInlineAsm()) {
3703 // For inline asm (either a call to asm function, or callbr instruction),
3704 // do the usual thing: check argument shadow and mark all outputs as
3705 // clean. Note that any side effects of the inline asm that are not
3706 // immediately visible in its constraints are not handled.
3707 if (ClHandleAsmConservative && MS.CompileKernel)
3708 visitAsmInstruction(CB);
3709 else
3710 visitInstruction(CB);
3711 return;
3712 }
3713 LibFunc LF;
3714 if (TLI->getLibFunc(CB, LF)) {
3715 // libatomic.a functions need to have special handling because there isn't
3716 // a good way to intercept them or compile the library with
3717 // instrumentation.
3718 switch (LF) {
3719 case LibFunc_atomic_load:
3720 if (!isa<CallInst>(CB)) {
3721 llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
3722 "Ignoring!\n";
3723 break;
3724 }
3725 visitLibAtomicLoad(CB);
3726 return;
3727 case LibFunc_atomic_store:
3728 visitLibAtomicStore(CB);
3729 return;
3730 default:
3731 break;
3732 }
3733 }
3734
3735 if (auto *Call = dyn_cast<CallInst>(&CB)) {
3736 assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere")(static_cast <bool> (!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere") ? void (0) : __assert_fail
("!isa<IntrinsicInst>(Call) && \"intrinsics are handled elsewhere\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3736
, __extension__ __PRETTY_FUNCTION__))
;
3737
3738 // We are going to insert code that relies on the fact that the callee
3739 // will become a non-readonly function after it is instrumented by us. To
3740 // prevent this code from being optimized out, mark that function
3741 // non-readonly in advance.
3742 AttributeMask B;
3743 B.addAttribute(Attribute::ReadOnly)
3744 .addAttribute(Attribute::ReadNone)
3745 .addAttribute(Attribute::WriteOnly)
3746 .addAttribute(Attribute::ArgMemOnly)
3747 .addAttribute(Attribute::Speculatable);
3748
3749 Call->removeFnAttrs(B);
3750 if (Function *Func = Call->getCalledFunction()) {
3751 Func->removeFnAttrs(B);
3752 }
3753
3754 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
3755 }
3756 IRBuilder<> IRB(&CB);
3757 bool MayCheckCall = MS.EagerChecks;
3758 if (Function *Func = CB.getCalledFunction()) {
3759 // __sanitizer_unaligned_{load,store} functions may be called by users
3760 // and always expects shadows in the TLS. So don't check them.
3761 MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
3762 }
3763
3764 unsigned ArgOffset = 0;
3765 LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " CallSite: " << CB <<
"\n"; } } while (false)
;
3766 for (const auto &[i, A] : llvm::enumerate(CB.args())) {
3767 if (!A->getType()->isSized()) {
3768 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Arg " << i << " is not sized: "
<< CB << "\n"; } } while (false)
;
3769 continue;
3770 }
3771 unsigned Size = 0;
3772 const DataLayout &DL = F.getParent()->getDataLayout();
3773
3774 bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
3775 bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
3776 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
3777
3778 if (EagerCheck) {
3779 insertShadowCheck(A, &CB);
3780 Size = DL.getTypeAllocSize(A->getType());
3781 } else {
3782 Value *Store = nullptr;
3783 // Compute the Shadow for arg even if it is ByVal, because
3784 // in that case getShadow() will copy the actual arg shadow to
3785 // __msan_param_tls.
3786 Value *ArgShadow = getShadow(A);
3787 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3788 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *Ado { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
3789 << " Shadow: " << *ArgShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Arg#" << i << ": "
<< *A << " Shadow: " << *ArgShadow <<
"\n"; } } while (false)
;
3790 if (ByVal) {
3791 // ByVal requires some special handling as it's too big for a single
3792 // load
3793 assert(A->getType()->isPointerTy() &&(static_cast <bool> (A->getType()->isPointerTy() &&
"ByVal argument is not a pointer!") ? void (0) : __assert_fail
("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3794
, __extension__ __PRETTY_FUNCTION__))
3794 "ByVal argument is not a pointer!")(static_cast <bool> (A->getType()->isPointerTy() &&
"ByVal argument is not a pointer!") ? void (0) : __assert_fail
("A->getType()->isPointerTy() && \"ByVal argument is not a pointer!\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3794
, __extension__ __PRETTY_FUNCTION__))
;
3795 Size = DL.getTypeAllocSize(CB.getParamByValType(i));
3796 if (ArgOffset + Size > kParamTLSSize)
3797 break;
3798 const MaybeAlign ParamAlignment(CB.getParamAlign(i));
3799 MaybeAlign Alignment = llvm::None;
3800 if (ParamAlignment)
3801 Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
3802 Value *AShadowPtr, *AOriginPtr;
3803 std::tie(AShadowPtr, AOriginPtr) =
3804 getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3805 /*isStore*/ false);
3806 if (!PropagateShadow) {
3807 Store = IRB.CreateMemSet(ArgShadowBase,
3808 Constant::getNullValue(IRB.getInt8Ty()),
3809 Size, Alignment);
3810 } else {
3811 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3812 Alignment, Size);
3813 if (MS.TrackOrigins) {
3814 Value *ArgOriginBase = getOriginPtrForArgument(A, IRB, ArgOffset);
3815 // FIXME: OriginSize should be:
3816 // alignTo(A % kMinOriginAlignment + Size, kMinOriginAlignment)
3817 unsigned OriginSize = alignTo(Size, kMinOriginAlignment);
3818 IRB.CreateMemCpy(
3819 ArgOriginBase,
3820 /* by origin_tls[ArgOffset] */ kMinOriginAlignment,
3821 AOriginPtr,
3822 /* by getShadowOriginPtr */ kMinOriginAlignment, OriginSize);
3823 }
3824 }
3825 } else {
3826 // Any other parameters mean we need bit-grained tracking of uninit
3827 // data
3828 Size = DL.getTypeAllocSize(A->getType());
3829 if (ArgOffset + Size > kParamTLSSize)
3830 break;
3831 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3832 kShadowTLSAlignment);
3833 Constant *Cst = dyn_cast<Constant>(ArgShadow);
3834 if (MS.TrackOrigins && !(Cst && Cst->isNullValue())) {
3835 IRB.CreateStore(getOrigin(A),
3836 getOriginPtrForArgument(A, IRB, ArgOffset));
3837 }
3838 }
3839 (void)Store;
3840 assert(Store != nullptr)(static_cast <bool> (Store != nullptr) ? void (0) : __assert_fail
("Store != nullptr", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3840, __extension__ __PRETTY_FUNCTION__))
;
3841 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Param:" << *Store <<
"\n"; } } while (false)
;
3842 }
3843 assert(Size != 0)(static_cast <bool> (Size != 0) ? void (0) : __assert_fail
("Size != 0", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 3843, __extension__ __PRETTY_FUNCTION__))
;
3844 ArgOffset += alignTo(Size, kShadowTLSAlignment);
3845 }
3846 LLVM_DEBUG(dbgs() << " done with call args\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " done with call args\n"; } } while
(false)
;
3847
3848 FunctionType *FT = CB.getFunctionType();
3849 if (FT->isVarArg()) {
3850 VAHelper->visitCallBase(CB, IRB);
3851 }
3852
3853 // Now, get the shadow for the RetVal.
3854 if (!CB.getType()->isSized())
3855 return;
3856 // Don't emit the epilogue for musttail call returns.
3857 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3858 return;
3859
3860 if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
3861 setShadow(&CB, getCleanShadow(&CB));
3862 setOrigin(&CB, getCleanOrigin());
3863 return;
3864 }
3865
3866 IRBuilder<> IRBBefore(&CB);
3867 // Until we have full dynamic coverage, make sure the retval shadow is 0.
3868 Value *Base = getShadowPtrForRetval(&CB, IRBBefore);
3869 IRBBefore.CreateAlignedStore(getCleanShadow(&CB), Base,
3870 kShadowTLSAlignment);
3871 BasicBlock::iterator NextInsn;
3872 if (isa<CallInst>(CB)) {
3873 NextInsn = ++CB.getIterator();
3874 assert(NextInsn != CB.getParent()->end())(static_cast <bool> (NextInsn != CB.getParent()->end
()) ? void (0) : __assert_fail ("NextInsn != CB.getParent()->end()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3874
, __extension__ __PRETTY_FUNCTION__))
;
3875 } else {
3876 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
3877 if (!NormalDest->getSinglePredecessor()) {
3878 // FIXME: this case is tricky, so we are just conservative here.
3879 // Perhaps we need to split the edge between this BB and NormalDest,
3880 // but a naive attempt to use SplitEdge leads to a crash.
3881 setShadow(&CB, getCleanShadow(&CB));
3882 setOrigin(&CB, getCleanOrigin());
3883 return;
3884 }
3885 // FIXME: NextInsn is likely in a basic block that has not been visited
3886 // yet. Anything inserted there will be instrumented by MSan later!
3887 NextInsn = NormalDest->getFirstInsertionPt();
3888 assert(NextInsn != NormalDest->end() &&(static_cast <bool> (NextInsn != NormalDest->end() &&
"Could not find insertion point for retval shadow load") ? void
(0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3889
, __extension__ __PRETTY_FUNCTION__))
3889 "Could not find insertion point for retval shadow load")(static_cast <bool> (NextInsn != NormalDest->end() &&
"Could not find insertion point for retval shadow load") ? void
(0) : __assert_fail ("NextInsn != NormalDest->end() && \"Could not find insertion point for retval shadow load\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 3889
, __extension__ __PRETTY_FUNCTION__))
;
3890 }
3891 IRBuilder<> IRBAfter(&*NextInsn);
3892 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3893 getShadowTy(&CB), getShadowPtrForRetval(&CB, IRBAfter),
3894 kShadowTLSAlignment, "_msret");
3895 setShadow(&CB, RetvalShadow);
3896 if (MS.TrackOrigins)
3897 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
3898 getOriginPtrForRetval(IRBAfter)));
3899 }
3900
3901 bool isAMustTailRetVal(Value *RetVal) {
3902 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3903 RetVal = I->getOperand(0);
3904 }
3905 if (auto *I = dyn_cast<CallInst>(RetVal)) {
3906 return I->isMustTailCall();
3907 }
3908 return false;
3909 }
3910
3911 void visitReturnInst(ReturnInst &I) {
3912 IRBuilder<> IRB(&I);
3913 Value *RetVal = I.getReturnValue();
3914 if (!RetVal)
3915 return;
3916 // Don't emit the epilogue for musttail call returns.
3917 if (isAMustTailRetVal(RetVal))
3918 return;
3919 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3920 bool HasNoUndef = F.hasRetAttribute(Attribute::NoUndef);
3921 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
3922 // FIXME: Consider using SpecialCaseList to specify a list of functions that
3923 // must always return fully initialized values. For now, we hardcode "main".
3924 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (F.getName() == "main");
3925
3926 Value *Shadow = getShadow(RetVal);
3927 bool StoreOrigin = true;
3928 if (EagerCheck) {
3929 insertShadowCheck(RetVal, &I);
3930 Shadow = getCleanShadow(RetVal);
3931 StoreOrigin = false;
3932 }
3933
3934 // The caller may still expect information passed over TLS if we pass our
3935 // check
3936 if (StoreShadow) {
3937 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3938 if (MS.TrackOrigins && StoreOrigin)
3939 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3940 }
3941 }
3942
3943 void visitPHINode(PHINode &I) {
3944 IRBuilder<> IRB(&I);
3945 if (!PropagateShadow) {
3946 setShadow(&I, getCleanShadow(&I));
3947 setOrigin(&I, getCleanOrigin());
3948 return;
3949 }
3950
3951 ShadowPHINodes.push_back(&I);
3952 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3953 "_msphi_s"));
3954 if (MS.TrackOrigins)
3955 setOrigin(
3956 &I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(), "_msphi_o"));
3957 }
3958
3959 Value *getLocalVarIdptr(AllocaInst &I) {
3960 ConstantInt *IntConst =
3961 ConstantInt::get(Type::getInt32Ty((*F.getParent()).getContext()), 0);
3962 return new GlobalVariable(*F.getParent(), IntConst->getType(),
3963 /*isConstant=*/false, GlobalValue::PrivateLinkage,
3964 IntConst);
3965 }
3966
3967 Value *getLocalVarDescription(AllocaInst &I) {
3968 return createPrivateConstGlobalForString(*F.getParent(), I.getName());
3969 }
3970
3971 void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3972 if (PoisonStack && ClPoisonStackWithCall) {
3973 IRB.CreateCall(MS.MsanPoisonStackFn,
3974 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3975 } else {
3976 Value *ShadowBase, *OriginBase;
3977 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
3978 &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
3979
3980 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3981 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlign());
3982 }
3983
3984 if (PoisonStack && MS.TrackOrigins) {
3985 Value *Idptr = getLocalVarIdptr(I);
3986 if (ClPrintStackNames) {
3987 Value *Descr = getLocalVarDescription(I);
3988 IRB.CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
3989 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3990 IRB.CreatePointerCast(Idptr, IRB.getInt8PtrTy()),
3991 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3992 } else {
3993 IRB.CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn,
3994 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3995 IRB.CreatePointerCast(Idptr, IRB.getInt8PtrTy())});
3996 }
3997 }
3998 }
3999
4000 void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
4001 Value *Descr = getLocalVarDescription(I);
4002 if (PoisonStack) {
4003 IRB.CreateCall(MS.MsanPoisonAllocaFn,
4004 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
4005 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
4006 } else {
4007 IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
4008 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
4009 }
4010 }
4011
4012 void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
4013 if (!InsPoint)
4014 InsPoint = &I;
4015 NextNodeIRBuilder IRB(InsPoint);
4016 const DataLayout &DL = F.getParent()->getDataLayout();
4017 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
4018 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
4019 if (I.isArrayAllocation())
4020 Len = IRB.CreateMul(Len,
4021 IRB.CreateZExtOrTrunc(I.getArraySize(), MS.IntptrTy));
4022
4023 if (MS.CompileKernel)
4024 poisonAllocaKmsan(I, IRB, Len);
4025 else
4026 poisonAllocaUserspace(I, IRB, Len);
4027 }
4028
4029 void visitAllocaInst(AllocaInst &I) {
4030 setShadow(&I, getCleanShadow(&I));
4031 setOrigin(&I, getCleanOrigin());
4032 // We'll get to this alloca later unless it's poisoned at the corresponding
4033 // llvm.lifetime.start.
4034 AllocaSet.insert(&I);
4035 }
4036
4037 void visitSelectInst(SelectInst &I) {
4038 IRBuilder<> IRB(&I);
4039 // a = select b, c, d
4040 Value *B = I.getCondition();
4041 Value *C = I.getTrueValue();
4042 Value *D = I.getFalseValue();
4043 Value *Sb = getShadow(B);
4044 Value *Sc = getShadow(C);
4045 Value *Sd = getShadow(D);
4046
4047 // Result shadow if condition shadow is 0.
4048 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
4049 Value *Sa1;
4050 if (I.getType()->isAggregateType()) {
4051 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
4052 // an extra "select". This results in much more compact IR.
4053 // Sa = select Sb, poisoned, (select b, Sc, Sd)
4054 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
4055 } else {
4056 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
4057 // If Sb (condition is poisoned), look for bits in c and d that are equal
4058 // and both unpoisoned.
4059 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
4060
4061 // Cast arguments to shadow-compatible type.
4062 C = CreateAppToShadowCast(IRB, C);
4063 D = CreateAppToShadowCast(IRB, D);
4064
4065 // Result shadow if condition shadow is 1.
4066 Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
4067 }
4068 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
4069 setShadow(&I, Sa);
4070 if (MS.TrackOrigins) {
4071 // Origins are always i32, so any vector conditions must be flattened.
4072 // FIXME: consider tracking vector origins for app vectors?
4073 if (B->getType()->isVectorTy()) {
4074 Type *FlatTy = getShadowTyNoVec(B->getType());
4075 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
4076 ConstantInt::getNullValue(FlatTy));
4077 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
4078 ConstantInt::getNullValue(FlatTy));
4079 }
4080 // a = select b, c, d
4081 // Oa = Sb ? Ob : (b ? Oc : Od)
4082 setOrigin(
4083 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
4084 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
4085 getOrigin(I.getFalseValue()))));
4086 }
4087 }
4088
4089 void visitLandingPadInst(LandingPadInst &I) {
4090 // Do nothing.
4091 // See https://github.com/google/sanitizers/issues/504
4092 setShadow(&I, getCleanShadow(&I));
4093 setOrigin(&I, getCleanOrigin());
4094 }
4095
4096 void visitCatchSwitchInst(CatchSwitchInst &I) {
4097 setShadow(&I, getCleanShadow(&I));
4098 setOrigin(&I, getCleanOrigin());
4099 }
4100
4101 void visitFuncletPadInst(FuncletPadInst &I) {
4102 setShadow(&I, getCleanShadow(&I));
4103 setOrigin(&I, getCleanOrigin());
4104 }
4105
4106 void visitGetElementPtrInst(GetElementPtrInst &I) { handleShadowOr(I); }
4107
4108 void visitExtractValueInst(ExtractValueInst &I) {
4109 IRBuilder<> IRB(&I);
4110 Value *Agg = I.getAggregateOperand();
4111 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "ExtractValue: " << I <<
"\n"; } } while (false)
;
4112 Value *AggShadow = getShadow(Agg);
4113 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
4114 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
4115 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " ResShadow: " << *ResShadow
<< "\n"; } } while (false)
;
4116 setShadow(&I, ResShadow);
4117 setOriginForNaryOp(I);
4118 }
4119
4120 void visitInsertValueInst(InsertValueInst &I) {
4121 IRBuilder<> IRB(&I);
4122 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "InsertValue: " << I <<
"\n"; } } while (false)
;
4123 Value *AggShadow = getShadow(I.getAggregateOperand());
4124 Value *InsShadow = getShadow(I.getInsertedValueOperand());
4125 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " AggShadow: " << *AggShadow
<< "\n"; } } while (false)
;
4126 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " InsShadow: " << *InsShadow
<< "\n"; } } while (false)
;
4127 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
4128 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << " Res: " << *Res <<
"\n"; } } while (false)
;
4129 setShadow(&I, Res);
4130 setOriginForNaryOp(I);
4131 }
4132
4133 void dumpInst(Instruction &I) {
4134 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
4135 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
4136 } else {
4137 errs() << "ZZZ " << I.getOpcodeName() << "\n";
4138 }
4139 errs() << "QQQ " << I << "\n";
4140 }
4141
4142 void visitResumeInst(ResumeInst &I) {
4143 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "Resume: " << I << "\n"
; } } while (false)
;
4144 // Nothing to do here.
4145 }
4146
4147 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
4148 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CleanupReturn: " << CRI <<
"\n"; } } while (false)
;
4149 // Nothing to do here.
4150 }
4151
4152 void visitCatchReturnInst(CatchReturnInst &CRI) {
4153 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "CatchReturn: " << CRI <<
"\n"; } } while (false)
;
4154 // Nothing to do here.
4155 }
4156
4157 void instrumentAsmArgument(Value *Operand, Type *ElemTy, Instruction &I,
4158 IRBuilder<> &IRB, const DataLayout &DL,
4159 bool isOutput) {
4160 // For each assembly argument, we check its value for being initialized.
4161 // If the argument is a pointer, we assume it points to a single element
4162 // of the corresponding type (or to a 8-byte word, if the type is unsized).
4163 // Each such pointer is instrumented with a call to the runtime library.
4164 Type *OpType = Operand->getType();
4165 // Check the operand value itself.
4166 insertShadowCheck(Operand, &I);
4167 if (!OpType->isPointerTy() || !isOutput) {
4168 assert(!isOutput)(static_cast <bool> (!isOutput) ? void (0) : __assert_fail
("!isOutput", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 4168, __extension__ __PRETTY_FUNCTION__))
;
4169 return;
4170 }
4171 if (!ElemTy->isSized())
4172 return;
4173 int Size = DL.getTypeStoreSize(ElemTy);
4174 Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
4175 Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
4176 IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
4177 }
4178
4179 /// Get the number of output arguments returned by pointers.
4180 int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
4181 int NumRetOutputs = 0;
4182 int NumOutputs = 0;
4183 Type *RetTy = cast<Value>(CB)->getType();
4184 if (!RetTy->isVoidTy()) {
4185 // Register outputs are returned via the CallInst return value.
4186 auto *ST = dyn_cast<StructType>(RetTy);
4187 if (ST)
4188 NumRetOutputs = ST->getNumElements();
4189 else
4190 NumRetOutputs = 1;
4191 }
4192 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
4193 for (const InlineAsm::ConstraintInfo &Info : Constraints) {
4194 switch (Info.Type) {
4195 case InlineAsm::isOutput:
4196 NumOutputs++;
4197 break;
4198 default:
4199 break;
4200 }
4201 }
4202 return NumOutputs - NumRetOutputs;
4203 }
4204
4205 void visitAsmInstruction(Instruction &I) {
4206 // Conservative inline assembly handling: check for poisoned shadow of
4207 // asm() arguments, then unpoison the result and all the memory locations
4208 // pointed to by those arguments.
4209 // An inline asm() statement in C++ contains lists of input and output
4210 // arguments used by the assembly code. These are mapped to operands of the
4211 // CallInst as follows:
4212 // - nR register outputs ("=r) are returned by value in a single structure
4213 // (SSA value of the CallInst);
4214 // - nO other outputs ("=m" and others) are returned by pointer as first
4215 // nO operands of the CallInst;
4216 // - nI inputs ("r", "m" and others) are passed to CallInst as the
4217 // remaining nI operands.
4218 // The total number of asm() arguments in the source is nR+nO+nI, and the
4219 // corresponding CallInst has nO+nI+1 operands (the last operand is the
4220 // function to be called).
4221 const DataLayout &DL = F.getParent()->getDataLayout();
4222 CallBase *CB = cast<CallBase>(&I);
4223 IRBuilder<> IRB(&I);
4224 InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
4225 int OutputArgs = getNumOutputArgs(IA, CB);
4226 // The last operand of a CallInst is the function itself.
4227 int NumOperands = CB->getNumOperands() - 1;
4228
4229 // Check input arguments. Doing so before unpoisoning output arguments, so
4230 // that we won't overwrite uninit values before checking them.
4231 for (int i = OutputArgs; i < NumOperands; i++) {
4232 Value *Operand = CB->getOperand(i);
4233 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4234 /*isOutput*/ false);
4235 }
4236 // Unpoison output arguments. This must happen before the actual InlineAsm
4237 // call, so that the shadow for memory published in the asm() statement
4238 // remains valid.
4239 for (int i = 0; i < OutputArgs; i++) {
4240 Value *Operand = CB->getOperand(i);
4241 instrumentAsmArgument(Operand, CB->getParamElementType(i), I, IRB, DL,
4242 /*isOutput*/ true);
4243 }
4244
4245 setShadow(&I, getCleanShadow(&I));
4246 setOrigin(&I, getCleanOrigin());
4247 }
4248
4249 void visitFreezeInst(FreezeInst &I) {
4250 // Freeze always returns a fully defined value.
4251 setShadow(&I, getCleanShadow(&I));
4252 setOrigin(&I, getCleanOrigin());
4253 }
4254
4255 void visitInstruction(Instruction &I) {
4256 // Everything else: stop propagating and check for poisoned shadow.
4257 if (ClDumpStrictInstructions)
4258 dumpInst(I);
4259 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("msan")) { dbgs() << "DEFAULT: " << I << "\n"
; } } while (false)
;
4260 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
4261 Value *Operand = I.getOperand(i);
4262 if (Operand->getType()->isSized())
4263 insertShadowCheck(Operand, &I);
4264 }
4265 setShadow(&I, getCleanShadow(&I));
4266 setOrigin(&I, getCleanOrigin());
4267 }
4268};
4269
4270/// AMD64-specific implementation of VarArgHelper.
4271struct VarArgAMD64Helper : public VarArgHelper {
4272 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
4273 // See a comment in visitCallBase for more details.
4274 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
4275 static const unsigned AMD64FpEndOffsetSSE = 176;
4276 // If SSE is disabled, fp_offset in va_list is zero.
4277 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4278
4279 unsigned AMD64FpEndOffset;
4280 Function &F;
4281 MemorySanitizer &MS;
4282 MemorySanitizerVisitor &MSV;
4283 Value *VAArgTLSCopy = nullptr;
4284 Value *VAArgTLSOriginCopy = nullptr;
4285 Value *VAArgOverflowSize = nullptr;
4286
4287 SmallVector<CallInst *, 16> VAStartInstrumentationList;
4288
4289 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4290
4291 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
4292 MemorySanitizerVisitor &MSV)
4293 : F(F), MS(MS), MSV(MSV) {
4294 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4295 for (const auto &Attr : F.getAttributes().getFnAttrs()) {
4296 if (Attr.isStringAttribute() &&
4297 (Attr.getKindAsString() == "target-features")) {
4298 if (Attr.getValueAsString().contains("-sse"))
4299 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4300 break;
4301 }
4302 }
4303 }
4304
4305 ArgKind classifyArgument(Value *arg) {
4306 // A very rough approximation of X86_64 argument classification rules.
4307 Type *T = arg->getType();
4308 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
4309 return AK_FloatingPoint;
4310 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4311 return AK_GeneralPurpose;
4312 if (T->isPointerTy())
4313 return AK_GeneralPurpose;
4314 return AK_Memory;
4315 }
4316
4317 // For VarArg functions, store the argument shadow in an ABI-specific format
4318 // that corresponds to va_list layout.
4319 // We do this because Clang lowers va_arg in the frontend, and this pass
4320 // only sees the low level code that deals with va_list internals.
4321 // A much easier alternative (provided that Clang emits va_arg instructions)
4322 // would have been to associate each live instance of va_list with a copy of
4323 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
4324 // order.
4325 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4326 unsigned GpOffset = 0;
4327 unsigned FpOffset = AMD64GpEndOffset;
4328 unsigned OverflowOffset = AMD64FpEndOffset;
4329 const DataLayout &DL = F.getParent()->getDataLayout();
4330 for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
4331 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4332 bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4333 if (IsByVal) {
4334 // ByVal arguments always go to the overflow area.
4335 // Fixed arguments passed through the overflow area will be stepped
4336 // over by va_start, so don't count them towards the offset.
4337 if (IsFixed)
4338 continue;
4339 assert(A->getType()->isPointerTy())(static_cast <bool> (A->getType()->isPointerTy())
? void (0) : __assert_fail ("A->getType()->isPointerTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4339
, __extension__ __PRETTY_FUNCTION__))
;
4340 Type *RealTy = CB.getParamByValType(ArgNo);
4341 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4342 Value *ShadowBase = getShadowPtrForVAArgument(
4343 RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
4344 Value *OriginBase = nullptr;
4345 if (MS.TrackOrigins)
4346 OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
4347 OverflowOffset += alignTo(ArgSize, 8);
4348 if (!ShadowBase)
4349 continue;
4350 Value *ShadowPtr, *OriginPtr;
4351 std::tie(ShadowPtr, OriginPtr) =
4352 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
4353 /*isStore*/ false);
4354
4355 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
4356 kShadowTLSAlignment, ArgSize);
4357 if (MS.TrackOrigins)
4358 IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
4359 kShadowTLSAlignment, ArgSize);
4360 } else {
4361 ArgKind AK = classifyArgument(A);
4362 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4363 AK = AK_Memory;
4364 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4365 AK = AK_Memory;
4366 Value *ShadowBase, *OriginBase = nullptr;
4367 switch (AK) {
4368 case AK_GeneralPurpose:
4369 ShadowBase =
4370 getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
4371 if (MS.TrackOrigins)
4372 OriginBase = getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
4373 GpOffset += 8;
4374 break;
4375 case AK_FloatingPoint:
4376 ShadowBase =
4377 getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
4378 if (MS.TrackOrigins)
4379 OriginBase = getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
4380 FpOffset += 16;
4381 break;
4382 case AK_Memory:
4383 if (IsFixed)
4384 continue;
4385 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4386 ShadowBase =
4387 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
4388 if (MS.TrackOrigins)
4389 OriginBase =
4390 getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
4391 OverflowOffset += alignTo(ArgSize, 8);
4392 }
4393 // Take fixed arguments into account for GpOffset and FpOffset,
4394 // but don't actually store shadows for them.
4395 // TODO(glider): don't call get*PtrForVAArgument() for them.
4396 if (IsFixed)
4397 continue;
4398 if (!ShadowBase)
4399 continue;
4400 Value *Shadow = MSV.getShadow(A);
4401 IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
4402 if (MS.TrackOrigins) {
4403 Value *Origin = MSV.getOrigin(A);
4404 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
4405 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4406 std::max(kShadowTLSAlignment, kMinOriginAlignment));
4407 }
4408 }
4409 }
4410 Constant *OverflowSize =
4411 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4412 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4413 }
4414
4415 /// Compute the shadow address for a given va_arg.
4416 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4417 unsigned ArgOffset, unsigned ArgSize) {
4418 // Make sure we don't overflow __msan_va_arg_tls.
4419 if (ArgOffset + ArgSize > kParamTLSSize)
4420 return nullptr;
4421 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4422 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4423 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4424 "_msarg_va_s");
4425 }
4426
4427 /// Compute the origin address for a given va_arg.
4428 Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
4429 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
4430 // getOriginPtrForVAArgument() is always called after
4431 // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
4432 // overflow.
4433 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4434 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
4435 "_msarg_va_o");
4436 }
4437
4438 void unpoisonVAListTagForInst(IntrinsicInst &I) {
4439 IRBuilder<> IRB(&I);
4440 Value *VAListTag = I.getArgOperand(0);
4441 Value *ShadowPtr, *OriginPtr;
4442 const Align Alignment = Align(8);
4443 std::tie(ShadowPtr, OriginPtr) =
4444 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
4445 /*isStore*/ true);
4446
4447 // Unpoison the whole __va_list_tag.
4448 // FIXME: magic ABI constants.
4449 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4450 /* size */ 24, Alignment, false);
4451 // We shouldn't need to zero out the origins, as they're only checked for
4452 // nonzero shadow.
4453 }
4454
4455 void visitVAStartInst(VAStartInst &I) override {
4456 if (F.getCallingConv() == CallingConv::Win64)
4457 return;
4458 VAStartInstrumentationList.push_back(&I);
4459 unpoisonVAListTagForInst(I);
4460 }
4461
4462 void visitVACopyInst(VACopyInst &I) override {
4463 if (F.getCallingConv() == CallingConv::Win64)
4464 return;
4465 unpoisonVAListTagForInst(I);
4466 }
4467
4468 void finalizeInstrumentation() override {
4469 assert(!VAArgOverflowSize && !VAArgTLSCopy &&(static_cast <bool> (!VAArgOverflowSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4470
, __extension__ __PRETTY_FUNCTION__))
4470 "finalizeInstrumentation called twice")(static_cast <bool> (!VAArgOverflowSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4470
, __extension__ __PRETTY_FUNCTION__))
;
4471 if (!VAStartInstrumentationList.empty()) {
4472 // If there is a va_start in this function, make a backup copy of
4473 // va_arg_tls somewhere in the function entry block.
4474 IRBuilder<> IRB(MSV.FnPrologueEnd);
4475 VAArgOverflowSize =
4476 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4477 Value *CopySize = IRB.CreateAdd(
4478 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
4479 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4480 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4481 if (MS.TrackOrigins) {
4482 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4483 IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
4484 Align(8), CopySize);
4485 }
4486 }
4487
4488 // Instrument va_start.
4489 // Copy va_list shadow from the backup copy of the TLS contents.
4490 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4491 CallInst *OrigInst = VAStartInstrumentationList[i];
4492 NextNodeIRBuilder IRB(OrigInst);
4493 Value *VAListTag = OrigInst->getArgOperand(0);
4494
4495 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4496 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
4497 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4498 ConstantInt::get(MS.IntptrTy, 16)),
4499 PointerType::get(RegSaveAreaPtrTy, 0));
4500 Value *RegSaveAreaPtr =
4501 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4502 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4503 const Align Alignment = Align(16);
4504 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4505 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4506 Alignment, /*isStore*/ true);
4507 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4508 AMD64FpEndOffset);
4509 if (MS.TrackOrigins)
4510 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4511 Alignment, AMD64FpEndOffset);
4512 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4513 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
4514 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4515 ConstantInt::get(MS.IntptrTy, 8)),
4516 PointerType::get(OverflowArgAreaPtrTy, 0));
4517 Value *OverflowArgAreaPtr =
4518 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4519 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4520 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4521 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
4522 Alignment, /*isStore*/ true);
4523 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
4524 AMD64FpEndOffset);
4525 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4526 VAArgOverflowSize);
4527 if (MS.TrackOrigins) {
4528 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
4529 AMD64FpEndOffset);
4530 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4531 VAArgOverflowSize);
4532 }
4533 }
4534 }
4535};
4536
4537/// MIPS64-specific implementation of VarArgHelper.
4538struct VarArgMIPS64Helper : public VarArgHelper {
4539 Function &F;
4540 MemorySanitizer &MS;
4541 MemorySanitizerVisitor &MSV;
4542 Value *VAArgTLSCopy = nullptr;
4543 Value *VAArgSize = nullptr;
4544
4545 SmallVector<CallInst *, 16> VAStartInstrumentationList;
4546
4547 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4548 MemorySanitizerVisitor &MSV)
4549 : F(F), MS(MS), MSV(MSV) {}
4550
4551 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4552 unsigned VAArgOffset = 0;
4553 const DataLayout &DL = F.getParent()->getDataLayout();
4554 for (Value *A :
4555 llvm::drop_begin(CB.args(), CB.getFunctionType()->getNumParams())) {
4556 Triple TargetTriple(F.getParent()->getTargetTriple());
4557 Value *Base;
4558 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4559 if (TargetTriple.getArch() == Triple::mips64) {
4560 // Adjusting the shadow for argument with size < 8 to match the
4561 // placement of bits in big endian system
4562 if (ArgSize < 8)
4563 VAArgOffset += (8 - ArgSize);
4564 }
4565 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4566 VAArgOffset += ArgSize;
4567 VAArgOffset = alignTo(VAArgOffset, 8);
4568 if (!Base)
4569 continue;
4570 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4571 }
4572
4573 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4574 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4575 // a new class member i.e. it is the total size of all VarArgs.
4576 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4577 }
4578
4579 /// Compute the shadow address for a given va_arg.
4580 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4581 unsigned ArgOffset, unsigned ArgSize) {
4582 // Make sure we don't overflow __msan_va_arg_tls.
4583 if (ArgOffset + ArgSize > kParamTLSSize)
4584 return nullptr;
4585 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4586 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4587 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4588 "_msarg");
4589 }
4590
4591 void visitVAStartInst(VAStartInst &I) override {
4592 IRBuilder<> IRB(&I);
4593 VAStartInstrumentationList.push_back(&I);
4594 Value *VAListTag = I.getArgOperand(0);
4595 Value *ShadowPtr, *OriginPtr;
4596 const Align Alignment = Align(8);
4597 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4598 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4599 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4600 /* size */ 8, Alignment, false);
4601 }
4602
4603 void visitVACopyInst(VACopyInst &I) override {
4604 IRBuilder<> IRB(&I);
4605 VAStartInstrumentationList.push_back(&I);
4606 Value *VAListTag = I.getArgOperand(0);
4607 Value *ShadowPtr, *OriginPtr;
4608 const Align Alignment = Align(8);
4609 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4610 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4611 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4612 /* size */ 8, Alignment, false);
4613 }
4614
4615 void finalizeInstrumentation() override {
4616 assert(!VAArgSize && !VAArgTLSCopy &&(static_cast <bool> (!VAArgSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4617
, __extension__ __PRETTY_FUNCTION__))
4617 "finalizeInstrumentation called twice")(static_cast <bool> (!VAArgSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4617
, __extension__ __PRETTY_FUNCTION__))
;
4618 IRBuilder<> IRB(MSV.FnPrologueEnd);
4619 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4620 Value *CopySize =
4621 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
4622
4623 if (!VAStartInstrumentationList.empty()) {
4624 // If there is a va_start in this function, make a backup copy of
4625 // va_arg_tls somewhere in the function entry block.
4626 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4627 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4628 }
4629
4630 // Instrument va_start.
4631 // Copy va_list shadow from the backup copy of the TLS contents.
4632 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4633 CallInst *OrigInst = VAStartInstrumentationList[i];
4634 NextNodeIRBuilder IRB(OrigInst);
4635 Value *VAListTag = OrigInst->getArgOperand(0);
4636 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4637 Value *RegSaveAreaPtrPtr =
4638 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4639 PointerType::get(RegSaveAreaPtrTy, 0));
4640 Value *RegSaveAreaPtr =
4641 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4642 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4643 const Align Alignment = Align(8);
4644 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4645 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4646 Alignment, /*isStore*/ true);
4647 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4648 CopySize);
4649 }
4650 }
4651};
4652
4653/// AArch64-specific implementation of VarArgHelper.
4654struct VarArgAArch64Helper : public VarArgHelper {
4655 static const unsigned kAArch64GrArgSize = 64;
4656 static const unsigned kAArch64VrArgSize = 128;
4657
4658 static const unsigned AArch64GrBegOffset = 0;
4659 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4660 // Make VR space aligned to 16 bytes.
4661 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4662 static const unsigned AArch64VrEndOffset =
4663 AArch64VrBegOffset + kAArch64VrArgSize;
4664 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4665
4666 Function &F;
4667 MemorySanitizer &MS;
4668 MemorySanitizerVisitor &MSV;
4669 Value *VAArgTLSCopy = nullptr;
4670 Value *VAArgOverflowSize = nullptr;
4671
4672 SmallVector<CallInst *, 16> VAStartInstrumentationList;
4673
4674 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4675
4676 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4677 MemorySanitizerVisitor &MSV)
4678 : F(F), MS(MS), MSV(MSV) {}
4679
4680 ArgKind classifyArgument(Value *arg) {
4681 Type *T = arg->getType();
4682 if (T->isFPOrFPVectorTy())
4683 return AK_FloatingPoint;
4684 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64) ||
4685 (T->isPointerTy()))
4686 return AK_GeneralPurpose;
4687 return AK_Memory;
4688 }
4689
4690 // The instrumentation stores the argument shadow in a non ABI-specific
4691 // format because it does not know which argument is named (since Clang,
4692 // like x86_64 case, lowers the va_args in the frontend and this pass only
4693 // sees the low level code that deals with va_list internals).
4694 // The first seven GR registers are saved in the first 56 bytes of the
4695 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4696 // the remaining arguments.
4697 // Using constant offset within the va_arg TLS array allows fast copy
4698 // in the finalize instrumentation.
4699 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4700 unsigned GrOffset = AArch64GrBegOffset;
4701 unsigned VrOffset = AArch64VrBegOffset;
4702 unsigned OverflowOffset = AArch64VAEndOffset;
4703
4704 const DataLayout &DL = F.getParent()->getDataLayout();
4705 for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
4706 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
1
Assuming the condition is false
4707 ArgKind AK = classifyArgument(A);
4708 if (AK
1.1
'AK' is not equal to AK_GeneralPurpose
== AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4709 AK = AK_Memory;
4710 if (AK
1.2
'AK' is equal to AK_FloatingPoint
== AK_FloatingPoint && VrOffset
1.3
'VrOffset' is < 'AArch64VrEndOffset'
>= AArch64VrEndOffset)
2
Taking false branch
4711 AK = AK_Memory;
4712 Value *Base;
4713 switch (AK) {
3
Control jumps to 'case AK_FloatingPoint:' at line 4718
4714 case AK_GeneralPurpose:
4715 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4716 GrOffset += 8;
4717 break;
4718 case AK_FloatingPoint:
4719 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4720 VrOffset += 16;
4721 break;
4722 case AK_Memory:
4723 // Don't count fixed arguments in the overflow area - va_start will
4724 // skip right over them.
4725 if (IsFixed)
4726 continue;
4727 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4728 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4729 alignTo(ArgSize, 8));
4730 OverflowOffset += alignTo(ArgSize, 8);
4731 break;
4732 }
4733 // Count Gp/Vr fixed arguments to their respective offsets, but don't
4734 // bother to actually store a shadow.
4735 if (IsFixed
4.1
'IsFixed' is false
)
4
Execution continues on line 4735
5
Taking false branch
4736 continue;
4737 if (!Base)
6
Assuming 'Base' is non-null
7
Taking false branch
4738 continue;
4739 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
8
Calling 'MemorySanitizerVisitor::getShadow'
4740 }
4741 Constant *OverflowSize =
4742 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4743 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4744 }
4745
4746 /// Compute the shadow address for a given va_arg.
4747 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4748 unsigned ArgOffset, unsigned ArgSize) {
4749 // Make sure we don't overflow __msan_va_arg_tls.
4750 if (ArgOffset + ArgSize > kParamTLSSize)
4751 return nullptr;
4752 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4753 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4754 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4755 "_msarg");
4756 }
4757
4758 void visitVAStartInst(VAStartInst &I) override {
4759 IRBuilder<> IRB(&I);
4760 VAStartInstrumentationList.push_back(&I);
4761 Value *VAListTag = I.getArgOperand(0);
4762 Value *ShadowPtr, *OriginPtr;
4763 const Align Alignment = Align(8);
4764 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4765 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4766 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4767 /* size */ 32, Alignment, false);
4768 }
4769
4770 void visitVACopyInst(VACopyInst &I) override {
4771 IRBuilder<> IRB(&I);
4772 VAStartInstrumentationList.push_back(&I);
4773 Value *VAListTag = I.getArgOperand(0);
4774 Value *ShadowPtr, *OriginPtr;
4775 const Align Alignment = Align(8);
4776 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4777 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4778 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4779 /* size */ 32, Alignment, false);
4780 }
4781
4782 // Retrieve a va_list field of 'void*' size.
4783 Value *getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4784 Value *SaveAreaPtrPtr = IRB.CreateIntToPtr(
4785 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4786 ConstantInt::get(MS.IntptrTy, offset)),
4787 Type::getInt64PtrTy(*MS.C));
4788 return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4789 }
4790
4791 // Retrieve a va_list field of 'int' size.
4792 Value *getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4793 Value *SaveAreaPtr = IRB.CreateIntToPtr(
4794 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4795 ConstantInt::get(MS.IntptrTy, offset)),
4796 Type::getInt32PtrTy(*MS.C));
4797 Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4798 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4799 }
4800
4801 void finalizeInstrumentation() override {
4802 assert(!VAArgOverflowSize && !VAArgTLSCopy &&(static_cast <bool> (!VAArgOverflowSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4803
, __extension__ __PRETTY_FUNCTION__))
4803 "finalizeInstrumentation called twice")(static_cast <bool> (!VAArgOverflowSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4803
, __extension__ __PRETTY_FUNCTION__))
;
4804 if (!VAStartInstrumentationList.empty()) {
4805 // If there is a va_start in this function, make a backup copy of
4806 // va_arg_tls somewhere in the function entry block.
4807 IRBuilder<> IRB(MSV.FnPrologueEnd);
4808 VAArgOverflowSize =
4809 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4810 Value *CopySize = IRB.CreateAdd(
4811 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
4812 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4813 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4814 }
4815
4816 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4817 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4818
4819 // Instrument va_start, copy va_list shadow from the backup copy of
4820 // the TLS contents.
4821 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4822 CallInst *OrigInst = VAStartInstrumentationList[i];
4823 NextNodeIRBuilder IRB(OrigInst);
4824
4825 Value *VAListTag = OrigInst->getArgOperand(0);
4826
4827 // The variadic ABI for AArch64 creates two areas to save the incoming
4828 // argument registers (one for 64-bit general register xn-x7 and another
4829 // for 128-bit FP/SIMD vn-v7).
4830 // We need then to propagate the shadow arguments on both regions
4831 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4832 // The remaining arguments are saved on shadow for 'va::stack'.
4833 // One caveat is it requires only to propagate the non-named arguments,
4834 // however on the call site instrumentation 'all' the arguments are
4835 // saved. So to copy the shadow values from the va_arg TLS array
4836 // we need to adjust the offset for both GR and VR fields based on
4837 // the __{gr,vr}_offs value (since they are stores based on incoming
4838 // named arguments).
4839
4840 // Read the stack pointer from the va_list.
4841 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4842
4843 // Read both the __gr_top and __gr_off and add them up.
4844 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4845 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4846
4847 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4848
4849 // Read both the __vr_top and __vr_off and add them up.
4850 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4851 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4852
4853 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4854
4855 // It does not know how many named arguments is being used and, on the
4856 // callsite all the arguments were saved. Since __gr_off is defined as
4857 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4858 // argument by ignoring the bytes of shadow from named arguments.
4859 Value *GrRegSaveAreaShadowPtrOff =
4860 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4861
4862 Value *GrRegSaveAreaShadowPtr =
4863 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4864 Align(8), /*isStore*/ true)
4865 .first;
4866
4867 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4868 GrRegSaveAreaShadowPtrOff);
4869 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4870
4871 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
4872 GrCopySize);
4873
4874 // Again, but for FP/SIMD values.
4875 Value *VrRegSaveAreaShadowPtrOff =
4876 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4877
4878 Value *VrRegSaveAreaShadowPtr =
4879 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4880 Align(8), /*isStore*/ true)
4881 .first;
4882
4883 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4884 IRB.getInt8Ty(),
4885 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4886 IRB.getInt32(AArch64VrBegOffset)),
4887 VrRegSaveAreaShadowPtrOff);
4888 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4889
4890 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
4891 VrCopySize);
4892
4893 // And finally for remaining arguments.
4894 Value *StackSaveAreaShadowPtr =
4895 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4896 Align(16), /*isStore*/ true)
4897 .first;
4898
4899 Value *StackSrcPtr = IRB.CreateInBoundsGEP(
4900 IRB.getInt8Ty(), VAArgTLSCopy, IRB.getInt32(AArch64VAEndOffset));
4901
4902 IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
4903 Align(16), VAArgOverflowSize);
4904 }
4905 }
4906};
4907
4908/// PowerPC64-specific implementation of VarArgHelper.
4909struct VarArgPowerPC64Helper : public VarArgHelper {
4910 Function &F;
4911 MemorySanitizer &MS;
4912 MemorySanitizerVisitor &MSV;
4913 Value *VAArgTLSCopy = nullptr;
4914 Value *VAArgSize = nullptr;
4915
4916 SmallVector<CallInst *, 16> VAStartInstrumentationList;
4917
4918 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4919 MemorySanitizerVisitor &MSV)
4920 : F(F), MS(MS), MSV(MSV) {}
4921
4922 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4923 // For PowerPC, we need to deal with alignment of stack arguments -
4924 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4925 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4926 // For that reason, we compute current offset from stack pointer (which is
4927 // always properly aligned), and offset for the first vararg, then subtract
4928 // them.
4929 unsigned VAArgBase;
4930 Triple TargetTriple(F.getParent()->getTargetTriple());
4931 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4932 // and 32 bytes for ABIv2. This is usually determined by target
4933 // endianness, but in theory could be overridden by function attribute.
4934 if (TargetTriple.getArch() == Triple::ppc64)
4935 VAArgBase = 48;
4936 else
4937 VAArgBase = 32;
4938 unsigned VAArgOffset = VAArgBase;
4939 const DataLayout &DL = F.getParent()->getDataLayout();
4940 for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
4941 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4942 bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4943 if (IsByVal) {
4944 assert(A->getType()->isPointerTy())(static_cast <bool> (A->getType()->isPointerTy())
? void (0) : __assert_fail ("A->getType()->isPointerTy()"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 4944
, __extension__ __PRETTY_FUNCTION__))
;
4945 Type *RealTy = CB.getParamByValType(ArgNo);
4946 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4947 Align ArgAlign = CB.getParamAlign(ArgNo).value_or(Align(8));
4948 if (ArgAlign < 8)
4949 ArgAlign = Align(8);
4950 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4951 if (!IsFixed) {
4952 Value *Base = getShadowPtrForVAArgument(
4953 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4954 if (Base) {
4955 Value *AShadowPtr, *AOriginPtr;
4956 std::tie(AShadowPtr, AOriginPtr) =
4957 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4958 kShadowTLSAlignment, /*isStore*/ false);
4959
4960 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4961 kShadowTLSAlignment, ArgSize);
4962 }
4963 }
4964 VAArgOffset += alignTo(ArgSize, Align(8));
4965 } else {
4966 Value *Base;
4967 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4968 Align ArgAlign = Align(8);
4969 if (A->getType()->isArrayTy()) {
4970 // Arrays are aligned to element size, except for long double
4971 // arrays, which are aligned to 8 bytes.
4972 Type *ElementTy = A->getType()->getArrayElementType();
4973 if (!ElementTy->isPPC_FP128Ty())
4974 ArgAlign = Align(DL.getTypeAllocSize(ElementTy));
4975 } else if (A->getType()->isVectorTy()) {
4976 // Vectors are naturally aligned.
4977 ArgAlign = Align(ArgSize);
4978 }
4979 if (ArgAlign < 8)
4980 ArgAlign = Align(8);
4981 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4982 if (DL.isBigEndian()) {
4983 // Adjusting the shadow for argument with size < 8 to match the
4984 // placement of bits in big endian system
4985 if (ArgSize < 8)
4986 VAArgOffset += (8 - ArgSize);
4987 }
4988 if (!IsFixed) {
4989 Base = getShadowPtrForVAArgument(A->getType(), IRB,
4990 VAArgOffset - VAArgBase, ArgSize);
4991 if (Base)
4992 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4993 }
4994 VAArgOffset += ArgSize;
4995 VAArgOffset = alignTo(VAArgOffset, Align(8));
4996 }
4997 if (IsFixed)
4998 VAArgBase = VAArgOffset;
4999 }
5000
5001 Constant *TotalVAArgSize =
5002 ConstantInt::get(IRB.getInt64Ty(), VAArgOffset - VAArgBase);
5003 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
5004 // a new class member i.e. it is the total size of all VarArgs.
5005 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5006 }
5007
5008 /// Compute the shadow address for a given va_arg.
5009 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
5010 unsigned ArgOffset, unsigned ArgSize) {
5011 // Make sure we don't overflow __msan_va_arg_tls.
5012 if (ArgOffset + ArgSize > kParamTLSSize)
5013 return nullptr;
5014 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
5015 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5016 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
5017 "_msarg");
5018 }
5019
5020 void visitVAStartInst(VAStartInst &I) override {
5021 IRBuilder<> IRB(&I);
5022 VAStartInstrumentationList.push_back(&I);
5023 Value *VAListTag = I.getArgOperand(0);
5024 Value *ShadowPtr, *OriginPtr;
5025 const Align Alignment = Align(8);
5026 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
5027 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
5028 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5029 /* size */ 8, Alignment, false);
5030 }
5031
5032 void visitVACopyInst(VACopyInst &I) override {
5033 IRBuilder<> IRB(&I);
5034 Value *VAListTag = I.getArgOperand(0);
5035 Value *ShadowPtr, *OriginPtr;
5036 const Align Alignment = Align(8);
5037 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
5038 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
5039 // Unpoison the whole __va_list_tag.
5040 // FIXME: magic ABI constants.
5041 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5042 /* size */ 8, Alignment, false);
5043 }
5044
5045 void finalizeInstrumentation() override {
5046 assert(!VAArgSize && !VAArgTLSCopy &&(static_cast <bool> (!VAArgSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 5047
, __extension__ __PRETTY_FUNCTION__))
5047 "finalizeInstrumentation called twice")(static_cast <bool> (!VAArgSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 5047
, __extension__ __PRETTY_FUNCTION__))
;
5048 IRBuilder<> IRB(MSV.FnPrologueEnd);
5049 VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5050 Value *CopySize =
5051 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5052
5053 if (!VAStartInstrumentationList.empty()) {
5054 // If there is a va_start in this function, make a backup copy of
5055 // va_arg_tls somewhere in the function entry block.
5056 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5057 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
5058 }
5059
5060 // Instrument va_start.
5061 // Copy va_list shadow from the backup copy of the TLS contents.
5062 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
5063 CallInst *OrigInst = VAStartInstrumentationList[i];
5064 NextNodeIRBuilder IRB(OrigInst);
5065 Value *VAListTag = OrigInst->getArgOperand(0);
5066 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5067 Value *RegSaveAreaPtrPtr =
5068 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5069 PointerType::get(RegSaveAreaPtrTy, 0));
5070 Value *RegSaveAreaPtr =
5071 IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5072 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5073 const Align Alignment = Align(8);
5074 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5075 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
5076 Alignment, /*isStore*/ true);
5077 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5078 CopySize);
5079 }
5080 }
5081};
5082
5083/// SystemZ-specific implementation of VarArgHelper.
5084struct VarArgSystemZHelper : public VarArgHelper {
5085 static const unsigned SystemZGpOffset = 16;
5086 static const unsigned SystemZGpEndOffset = 56;
5087 static const unsigned SystemZFpOffset = 128;
5088 static const unsigned SystemZFpEndOffset = 160;
5089 static const unsigned SystemZMaxVrArgs = 8;
5090 static const unsigned SystemZRegSaveAreaSize = 160;
5091 static const unsigned SystemZOverflowOffset = 160;
5092 static const unsigned SystemZVAListTagSize = 32;
5093 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5094 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5095
5096 Function &F;
5097 MemorySanitizer &MS;
5098 MemorySanitizerVisitor &MSV;
5099 Value *VAArgTLSCopy = nullptr;
5100 Value *VAArgTLSOriginCopy = nullptr;
5101 Value *VAArgOverflowSize = nullptr;
5102
5103 SmallVector<CallInst *, 16> VAStartInstrumentationList;
5104
5105 enum class ArgKind {
5106 GeneralPurpose,
5107 FloatingPoint,
5108 Vector,
5109 Memory,
5110 Indirect,
5111 };
5112
5113 enum class ShadowExtension { None, Zero, Sign };
5114
5115 VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
5116 MemorySanitizerVisitor &MSV)
5117 : F(F), MS(MS), MSV(MSV) {}
5118
5119 ArgKind classifyArgument(Type *T, bool IsSoftFloatABI) {
5120 // T is a SystemZABIInfo::classifyArgumentType() output, and there are
5121 // only a few possibilities of what it can be. In particular, enums, single
5122 // element structs and large types have already been taken care of.
5123
5124 // Some i128 and fp128 arguments are converted to pointers only in the
5125 // back end.
5126 if (T->isIntegerTy(128) || T->isFP128Ty())
5127 return ArgKind::Indirect;
5128 if (T->isFloatingPointTy())
5129 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5130 if (T->isIntegerTy() || T->isPointerTy())
5131 return ArgKind::GeneralPurpose;
5132 if (T->isVectorTy())
5133 return ArgKind::Vector;
5134 return ArgKind::Memory;
5135 }
5136
5137 ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
5138 // ABI says: "One of the simple integer types no more than 64 bits wide.
5139 // ... If such an argument is shorter than 64 bits, replace it by a full
5140 // 64-bit integer representing the same number, using sign or zero
5141 // extension". Shadow for an integer argument has the same type as the
5142 // argument itself, so it can be sign or zero extended as well.
5143 bool ZExt = CB.paramHasAttr(ArgNo, Attribute::ZExt);
5144 bool SExt = CB.paramHasAttr(ArgNo, Attribute::SExt);
5145 if (ZExt) {
5146 assert(!SExt)(static_cast <bool> (!SExt) ? void (0) : __assert_fail (
"!SExt", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 5146, __extension__ __PRETTY_FUNCTION__))
;
5147 return ShadowExtension::Zero;
5148 }
5149 if (SExt) {
5150 assert(!ZExt)(static_cast <bool> (!ZExt) ? void (0) : __assert_fail (
"!ZExt", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 5150, __extension__ __PRETTY_FUNCTION__))
;
5151 return ShadowExtension::Sign;
5152 }
5153 return ShadowExtension::None;
5154 }
5155
5156 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5157 bool IsSoftFloatABI = CB.getCalledFunction()
5158 ->getFnAttribute("use-soft-float")
5159 .getValueAsBool();
5160 unsigned GpOffset = SystemZGpOffset;
5161 unsigned FpOffset = SystemZFpOffset;
5162 unsigned VrIndex = 0;
5163 unsigned OverflowOffset = SystemZOverflowOffset;
5164 const DataLayout &DL = F.getParent()->getDataLayout();
5165 for (const auto &[ArgNo, A] : llvm::enumerate(CB.args())) {
5166 bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5167 // SystemZABIInfo does not produce ByVal parameters.
5168 assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal))(static_cast <bool> (!CB.paramHasAttr(ArgNo, Attribute::
ByVal)) ? void (0) : __assert_fail ("!CB.paramHasAttr(ArgNo, Attribute::ByVal)"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 5168
, __extension__ __PRETTY_FUNCTION__))
;
5169 Type *T = A->getType();
5170 ArgKind AK = classifyArgument(T, IsSoftFloatABI);
5171 if (AK == ArgKind::Indirect) {
5172 T = PointerType::get(T, 0);
5173 AK = ArgKind::GeneralPurpose;
5174 }
5175 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5176 AK = ArgKind::Memory;
5177 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5178 AK = ArgKind::Memory;
5179 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5180 AK = ArgKind::Memory;
5181 Value *ShadowBase = nullptr;
5182 Value *OriginBase = nullptr;
5183 ShadowExtension SE = ShadowExtension::None;
5184 switch (AK) {
5185 case ArgKind::GeneralPurpose: {
5186 // Always keep track of GpOffset, but store shadow only for varargs.
5187 uint64_t ArgSize = 8;
5188 if (GpOffset + ArgSize <= kParamTLSSize) {
5189 if (!IsFixed) {
5190 SE = getShadowExtension(CB, ArgNo);
5191 uint64_t GapSize = 0;
5192 if (SE == ShadowExtension::None) {
5193 uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5194 assert(ArgAllocSize <= ArgSize)(static_cast <bool> (ArgAllocSize <= ArgSize) ? void
(0) : __assert_fail ("ArgAllocSize <= ArgSize", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 5194, __extension__ __PRETTY_FUNCTION__))
;
5195 GapSize = ArgSize - ArgAllocSize;
5196 }
5197 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5198 if (MS.TrackOrigins)
5199 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5200 }
5201 GpOffset += ArgSize;
5202 } else {
5203 GpOffset = kParamTLSSize;
5204 }
5205 break;
5206 }
5207 case ArgKind::FloatingPoint: {
5208 // Always keep track of FpOffset, but store shadow only for varargs.
5209 uint64_t ArgSize = 8;
5210 if (FpOffset + ArgSize <= kParamTLSSize) {
5211 if (!IsFixed) {
5212 // PoP says: "A short floating-point datum requires only the
5213 // left-most 32 bit positions of a floating-point register".
5214 // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
5215 // don't extend shadow and don't mind the gap.
5216 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5217 if (MS.TrackOrigins)
5218 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5219 }
5220 FpOffset += ArgSize;
5221 } else {
5222 FpOffset = kParamTLSSize;
5223 }
5224 break;
5225 }
5226 case ArgKind::Vector: {
5227 // Keep track of VrIndex. No need to store shadow, since vector varargs
5228 // go through AK_Memory.
5229 assert(IsFixed)(static_cast <bool> (IsFixed) ? void (0) : __assert_fail
("IsFixed", "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp"
, 5229, __extension__ __PRETTY_FUNCTION__))
;
5230 VrIndex++;
5231 break;
5232 }
5233 case ArgKind::Memory: {
5234 // Keep track of OverflowOffset and store shadow only for varargs.
5235 // Ignore fixed args, since we need to copy only the vararg portion of
5236 // the overflow area shadow.
5237 if (!IsFixed) {
5238 uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5239 uint64_t ArgSize = alignTo(ArgAllocSize, 8);
5240 if (OverflowOffset + ArgSize <= kParamTLSSize) {
5241 SE = getShadowExtension(CB, ArgNo);
5242 uint64_t GapSize =
5243 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5244 ShadowBase =
5245 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5246 if (MS.TrackOrigins)
5247 OriginBase =
5248 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5249 OverflowOffset += ArgSize;
5250 } else {
5251 OverflowOffset = kParamTLSSize;
5252 }
5253 }
5254 break;
5255 }
5256 case ArgKind::Indirect:
5257 llvm_unreachable("Indirect must be converted to GeneralPurpose")::llvm::llvm_unreachable_internal("Indirect must be converted to GeneralPurpose"
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 5257
)
;
5258 }
5259 if (ShadowBase == nullptr)
5260 continue;
5261 Value *Shadow = MSV.getShadow(A);
5262 if (SE != ShadowExtension::None)
5263 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.getInt64Ty(),
5264 /*Signed*/ SE == ShadowExtension::Sign);
5265 ShadowBase = IRB.CreateIntToPtr(
5266 ShadowBase, PointerType::get(Shadow->getType(), 0), "_msarg_va_s");
5267 IRB.CreateStore(Shadow, ShadowBase);
5268 if (MS.TrackOrigins) {
5269 Value *Origin = MSV.getOrigin(A);
5270 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
5271 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5272 kMinOriginAlignment);
5273 }
5274 }
5275 Constant *OverflowSize = ConstantInt::get(
5276 IRB.getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5277 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5278 }
5279
5280 Value *getShadowAddrForVAArgument(IRBuilder<> &IRB, unsigned ArgOffset) {
5281 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
5282 return IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5283 }
5284
5285 Value *getOriginPtrForVAArgument(IRBuilder<> &IRB, int ArgOffset) {
5286 Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
5287 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
5288 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
5289 "_msarg_va_o");
5290 }
5291
5292 void unpoisonVAListTagForInst(IntrinsicInst &I) {
5293 IRBuilder<> IRB(&I);
5294 Value *VAListTag = I.getArgOperand(0);
5295 Value *ShadowPtr, *OriginPtr;
5296 const Align Alignment = Align(8);
5297 std::tie(ShadowPtr, OriginPtr) =
5298 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
5299 /*isStore*/ true);
5300 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
5301 SystemZVAListTagSize, Alignment, false);
5302 }
5303
5304 void visitVAStartInst(VAStartInst &I) override {
5305 VAStartInstrumentationList.push_back(&I);
5306 unpoisonVAListTagForInst(I);
5307 }
5308
5309 void visitVACopyInst(VACopyInst &I) override { unpoisonVAListTagForInst(I); }
5310
5311 void copyRegSaveArea(IRBuilder<> &IRB, Value *VAListTag) {
5312 Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5313 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
5314 IRB.CreateAdd(
5315 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5316 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5317 PointerType::get(RegSaveAreaPtrTy, 0));
5318 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5319 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5320 const Align Alignment = Align(8);
5321 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5322 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(), Alignment,
5323 /*isStore*/ true);
5324 // TODO(iii): copy only fragments filled by visitCallBase()
5325 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5326 SystemZRegSaveAreaSize);
5327 if (MS.TrackOrigins)
5328 IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5329 Alignment, SystemZRegSaveAreaSize);
5330 }
5331
5332 void copyOverflowArea(IRBuilder<> &IRB, Value *VAListTag) {
5333 Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
5334 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
5335 IRB.CreateAdd(
5336 IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
5337 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5338 PointerType::get(OverflowArgAreaPtrTy, 0));
5339 Value *OverflowArgAreaPtr =
5340 IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5341 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5342 const Align Alignment = Align(8);
5343 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5344 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
5345 Alignment, /*isStore*/ true);
5346 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
5347 SystemZOverflowOffset);
5348 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5349 VAArgOverflowSize);
5350 if (MS.TrackOrigins) {
5351 SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
5352 SystemZOverflowOffset);
5353 IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5354 VAArgOverflowSize);
5355 }
5356 }
5357
5358 void finalizeInstrumentation() override {
5359 assert(!VAArgOverflowSize && !VAArgTLSCopy &&(static_cast <bool> (!VAArgOverflowSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 5360
, __extension__ __PRETTY_FUNCTION__))
5360 "finalizeInstrumentation called twice")(static_cast <bool> (!VAArgOverflowSize && !VAArgTLSCopy
&& "finalizeInstrumentation called twice") ? void (0
) : __assert_fail ("!VAArgOverflowSize && !VAArgTLSCopy && \"finalizeInstrumentation called twice\""
, "llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp", 5360
, __extension__ __PRETTY_FUNCTION__))
;
5361 if (!VAStartInstrumentationList.empty()) {
5362 // If there is a va_start in this function, make a backup copy of
5363 // va_arg_tls somewhere in the function entry block.
5364 IRBuilder<> IRB(MSV.FnPrologueEnd);
5365 VAArgOverflowSize =
5366 IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
5367 Value *CopySize =
5368 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5369 VAArgOverflowSize);
5370 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5371 IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
5372 if (MS.TrackOrigins) {
5373 VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
5374 IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
5375 Align(8), CopySize);
5376 }
5377 }
5378
5379 // Instrument va_start.
5380 // Copy va_list shadow from the backup copy of the TLS contents.
5381 for (size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.size();
5382 VaStartNo < VaStartNum; VaStartNo++) {
5383 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5384 NextNodeIRBuilder IRB(OrigInst);
5385 Value *VAListTag = OrigInst->getArgOperand(0);
5386 copyRegSaveArea(IRB, VAListTag);
5387 copyOverflowArea(IRB, VAListTag);
5388 }
5389 }
5390};
5391
5392/// A no-op implementation of VarArgHelper.
5393struct VarArgNoOpHelper : public VarArgHelper {
5394 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
5395 MemorySanitizerVisitor &MSV) {}
5396
5397 void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {}
5398
5399 void visitVAStartInst(VAStartInst &I) override {}
5400
5401 void visitVACopyInst(VACopyInst &I) override {}
5402
5403 void finalizeInstrumentation() override {}
5404};
5405
5406} // end anonymous namespace
5407
5408static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
5409 MemorySanitizerVisitor &Visitor) {
5410 // VarArg handling is only implemented on AMD64. False positives are possible
5411 // on other platforms.
5412 Triple TargetTriple(Func.getParent()->getTargetTriple());
5413 if (TargetTriple.getArch() == Triple::x86_64)
5414 return new VarArgAMD64Helper(Func, Msan, Visitor);
5415 else if (TargetTriple.isMIPS64())
5416 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5417 else if (TargetTriple.getArch() == Triple::aarch64)
5418 return new VarArgAArch64Helper(Func, Msan, Visitor);
5419 else if (TargetTriple.getArch() == Triple::ppc64 ||
5420 TargetTriple.getArch() == Triple::ppc64le)
5421 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5422 else if (TargetTriple.getArch() == Triple::systemz)
5423 return new VarArgSystemZHelper(Func, Msan, Visitor);
5424 else
5425 return new VarArgNoOpHelper(Func, Msan, Visitor);
5426}
5427
5428bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
5429 if (!CompileKernel && F.getName() == kMsanModuleCtorName)
5430 return false;
5431
5432 if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5433 return false;
5434
5435 MemorySanitizerVisitor Visitor(F, *this, TLI);
5436
5437 // Clear out readonly/readnone attributes.
5438 AttributeMask B;
5439 B.addAttribute(Attribute::ReadOnly)
5440 .addAttribute(Attribute::ReadNone)
5441 .addAttribute(Attribute::WriteOnly)
5442 .addAttribute(Attribute::ArgMemOnly)
5443 .addAttribute(Attribute::Speculatable);
5444 F.removeFnAttrs(B);
5445
5446 return Visitor.runOnFunction();
5447}