184#include "llvm/IR/IntrinsicsX86.h"
213#define DEBUG_TYPE "msan"
216 "Controls which checks to insert");
219 "Controls which instruction to instrument");
237 "msan-track-origins",
242 cl::desc(
"keep going after reporting a UMR"),
251 "msan-poison-stack-with-call",
256 "msan-poison-stack-pattern",
257 cl::desc(
"poison uninitialized stack variables with the given pattern"),
262 cl::desc(
"Print name of local stack variable"),
271 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
276 cl::desc(
"exact handling of relational integer ICmp"),
280 "msan-handle-lifetime-intrinsics",
282 "when possible, poison scoped variables at the beginning of the scope "
283 "(slower, but more precise)"),
294 "msan-handle-asm-conservative",
305 "msan-check-access-address",
306 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
311 cl::desc(
"check arguments and return values at function call boundaries"),
315 "msan-dump-strict-instructions",
316 cl::desc(
"print out instructions with default strict semantics"),
320 "msan-instrumentation-with-call-threshold",
322 "If the function being instrumented requires more than "
323 "this number of checks and origin stores, use callbacks instead of "
324 "inline checks (-1 means never use callbacks)."),
329 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
339 cl::desc(
"Insert checks for constant shadow values"),
346 cl::desc(
"Place MSan constructors in comdat sections"),
352 cl::desc(
"Define custom MSan AndMask"),
356 cl::desc(
"Define custom MSan XorMask"),
360 cl::desc(
"Define custom MSan ShadowBase"),
364 cl::desc(
"Define custom MSan OriginBase"),
369 cl::desc(
"Define threshold for number of checks per "
370 "debug location to force origin update."),
382struct MemoryMapParams {
389struct PlatformMemoryMapParams {
390 const MemoryMapParams *bits32;
391 const MemoryMapParams *bits64;
537class MemorySanitizer {
546 MemorySanitizer(MemorySanitizer &&) =
delete;
547 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
548 MemorySanitizer(
const MemorySanitizer &) =
delete;
549 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
554 friend struct MemorySanitizerVisitor;
555 friend struct VarArgHelperBase;
556 friend struct VarArgAMD64Helper;
557 friend struct VarArgMIPS64Helper;
558 friend struct VarArgAArch64Helper;
559 friend struct VarArgPowerPC64Helper;
560 friend struct VarArgSystemZHelper;
562 void initializeModule(
Module &M);
567 template <
typename... ArgsTy>
594 Value *ParamOriginTLS;
600 Value *RetvalOriginTLS;
606 Value *VAArgOriginTLS;
609 Value *VAArgOverflowSizeTLS;
612 bool CallbacksInitialized =
false;
657 Value *MsanMetadataAlloca;
663 const MemoryMapParams *MapParams;
667 MemoryMapParams CustomMapParams;
672 MDNode *OriginStoreWeights;
675void insertModuleCtor(
Module &M) {
703 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
718 MemorySanitizer Msan(*
F.getParent(),
Options);
737 OS, MapClassName2PassName);
744 OS <<
"eager-checks;";
745 OS <<
"track-origins=" <<
Options.TrackOrigins;
761template <
typename... ArgsTy>
769 std::forward<ArgsTy>(Args)...);
772 return M.getOrInsertFunction(
Name, MsanMetadata,
773 std::forward<ArgsTy>(Args)...);
782 RetvalOriginTLS =
nullptr;
784 ParamOriginTLS =
nullptr;
786 VAArgOriginTLS =
nullptr;
787 VAArgOverflowSizeTLS =
nullptr;
789 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
791 IRB.getVoidTy(), IRB.getInt32Ty());
802 MsanGetContextStateFn =
M.getOrInsertFunction(
808 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
809 std::string name_load =
810 "__msan_metadata_ptr_for_load_" + std::to_string(size);
811 std::string name_store =
812 "__msan_metadata_ptr_for_store_" + std::to_string(size);
813 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
815 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
819 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
822 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
823 M,
"__msan_metadata_ptr_for_store_n",
827 MsanPoisonAllocaFn =
M.getOrInsertFunction(
828 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
829 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
830 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
834 return M.getOrInsertGlobal(
Name, Ty, [&] {
836 nullptr,
Name,
nullptr,
849 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
850 :
"__msan_warning_with_origin_noreturn";
851 WarningFn =
M.getOrInsertFunction(WarningFnName,
853 IRB.getVoidTy(), IRB.getInt32Ty());
856 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
857 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
883 VAArgOverflowSizeTLS =
888 unsigned AccessSize = 1 << AccessSizeIndex;
889 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
890 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
892 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
894 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
895 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
897 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
901 MsanSetAllocaOriginWithDescriptionFn =
902 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
903 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
904 MsanSetAllocaOriginNoDescriptionFn =
905 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
906 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
907 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
908 IRB.getVoidTy(), PtrTy, IntptrTy);
914 if (CallbacksInitialized)
920 MsanChainOriginFn =
M.getOrInsertFunction(
921 "__msan_chain_origin",
924 MsanSetOriginFn =
M.getOrInsertFunction(
926 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
928 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
930 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
931 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
933 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
935 MsanInstrumentAsmStoreFn =
936 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
940 createKernelApi(M, TLI);
942 createUserspaceApi(M, TLI);
944 CallbacksInitialized =
true;
950 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
968void MemorySanitizer::initializeModule(
Module &M) {
969 auto &
DL =
M.getDataLayout();
971 TargetTriple =
Triple(
M.getTargetTriple());
973 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
974 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
976 if (ShadowPassed || OriginPassed) {
981 MapParams = &CustomMapParams;
983 switch (TargetTriple.getOS()) {
985 switch (TargetTriple.getArch()) {
1000 switch (TargetTriple.getArch()) {
1009 switch (TargetTriple.getArch()) {
1043 C = &(
M.getContext());
1045 IntptrTy = IRB.getIntPtrTy(
DL);
1046 OriginTy = IRB.getInt32Ty();
1047 PtrTy = IRB.getPtrTy();
1052 if (!CompileKernel) {
1054 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1055 return new GlobalVariable(
1056 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1057 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1061 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1062 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1063 GlobalValue::WeakODRLinkage,
1064 IRB.getInt32(Recover),
"__msan_keep_going");
1079struct VarArgHelper {
1080 virtual ~VarArgHelper() =
default;
1095 virtual void finalizeInstrumentation() = 0;
1098struct MemorySanitizerVisitor;
1103 MemorySanitizerVisitor &Visitor);
1110 if (TypeSizeFixed <= 8)
1119class NextNodeIRBuilder :
public IRBuilder<> {
1132struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1134 MemorySanitizer &MS;
1137 std::unique_ptr<VarArgHelper> VAHelper;
1145 bool PropagateShadow;
1149 struct ShadowOriginAndInsertPoint {
1155 : Shadow(S), Origin(
O), OrigIns(
I) {}
1163 int64_t SplittableBlocksCount = 0;
1165 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1168 bool SanitizeFunction =
1170 InsertChecks = SanitizeFunction;
1171 PropagateShadow = SanitizeFunction;
1181 MS.initializeCallbacks(*
F.getParent(), TLI);
1182 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1185 if (MS.CompileKernel) {
1187 insertKmsanPrologue(IRB);
1191 <<
"MemorySanitizer is not inserting checks into '"
1192 <<
F.getName() <<
"'\n");
1195 bool instrumentWithCalls(
Value *V) {
1197 if (isa<Constant>(V))
1200 ++SplittableBlocksCount;
1206 return I.getParent() == FnPrologueEnd->
getParent() &&
1207 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1215 if (MS.TrackOrigins <= 1)
1217 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1222 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1234 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1235 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1247 auto [InsertPt,
Index] =
1259 Align CurrentAlignment = Alignment;
1260 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1261 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1262 Value *IntptrOriginPtr =
1264 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1269 CurrentAlignment = IntptrAlignment;
1286 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1287 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1295 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1304 if (instrumentWithCalls(ConvertedShadow) &&
1307 Value *ConvertedShadow2 =
1313 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1317 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1322 void materializeStores() {
1325 Value *Val =
SI->getValueOperand();
1327 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1328 Value *ShadowPtr, *OriginPtr;
1330 const Align Alignment =
SI->getAlign();
1332 std::tie(ShadowPtr, OriginPtr) =
1333 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1342 if (MS.TrackOrigins && !
SI->isAtomic())
1343 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1350 if (MS.TrackOrigins < 2)
1353 if (LazyWarningDebugLocationCount.
empty())
1354 for (
const auto &
I : InstrumentationList)
1355 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1369 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1371 auto NewDebugLoc = OI->getDebugLoc();
1378 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1379 Origin = updateOrigin(Origin, IRBOrigin);
1384 if (MS.CompileKernel || MS.TrackOrigins)
1398 if (instrumentWithCalls(ConvertedShadow) &&
1401 Value *ConvertedShadow2 =
1404 Fn, {ConvertedShadow2,
1405 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1409 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1412 !MS.Recover, MS.ColdCallWeights);
1415 insertWarningFn(IRB, Origin);
1420 void materializeInstructionChecks(
1425 bool Combine = !MS.TrackOrigins;
1427 Value *Shadow =
nullptr;
1428 for (
const auto &ShadowData : InstructionChecks) {
1432 Value *ConvertedShadow = ShadowData.Shadow;
1434 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1441 insertWarningFn(IRB, ShadowData.Origin);
1451 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1456 Shadow = ConvertedShadow;
1460 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1461 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1462 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1468 materializeOneCheck(IRB, Shadow,
nullptr);
1472 void materializeChecks() {
1478 for (
auto I = InstrumentationList.begin();
1479 I != InstrumentationList.end();) {
1480 auto OrigIns =
I->OrigIns;
1484 auto J = std::find_if(
I + 1, InstrumentationList.end(),
1485 [OrigIns](
const ShadowOriginAndInsertPoint &R) {
1486 return OrigIns != R.OrigIns;
1500 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1501 {Zero, IRB.getInt32(0)},
"param_shadow");
1502 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1503 {Zero, IRB.getInt32(1)},
"retval_shadow");
1504 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1505 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1506 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1507 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1508 MS.VAArgOverflowSizeTLS =
1509 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1510 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1511 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1512 {Zero, IRB.getInt32(5)},
"param_origin");
1513 MS.RetvalOriginTLS =
1514 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1515 {Zero, IRB.getInt32(6)},
"retval_origin");
1517 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1534 for (
PHINode *PN : ShadowPHINodes) {
1535 PHINode *PNS = cast<PHINode>(getShadow(PN));
1536 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1537 size_t NumValues = PN->getNumIncomingValues();
1538 for (
size_t v = 0;
v < NumValues;
v++) {
1539 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1541 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1545 VAHelper->finalizeInstrumentation();
1549 if (InstrumentLifetimeStart) {
1550 for (
auto Item : LifetimeStartList) {
1551 instrumentAlloca(*Item.second, Item.first);
1552 AllocaSet.
remove(Item.second);
1558 instrumentAlloca(*AI);
1561 materializeChecks();
1565 materializeStores();
1571 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1583 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1584 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1586 VT->getElementCount());
1588 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1589 return ArrayType::get(getShadowTy(AT->getElementType()),
1590 AT->getNumElements());
1592 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1594 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1595 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1597 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1613 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1615 if (Aggregator != FalseVal)
1616 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1618 Aggregator = ShadowBool;
1627 if (!
Array->getNumElements())
1631 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1635 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1636 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1646 return collapseStructShadow(
Struct, V, IRB);
1647 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1648 return collapseArrayShadow(Array, V, IRB);
1649 if (isa<VectorType>(
V->getType())) {
1650 if (isa<ScalableVectorType>(
V->getType()))
1653 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1661 Type *VTy =
V->getType();
1663 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1670 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1671 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1672 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1673 VectTy->getElementCount());
1679 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1680 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1681 return VectorType::get(
1682 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1683 VectTy->getElementCount());
1685 assert(IntPtrTy == MS.IntptrTy);
1686 return PointerType::get(*MS.C, 0);
1690 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1692 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1694 assert(IntPtrTy == MS.IntptrTy);
1695 return ConstantInt::get(MS.IntptrTy,
C);
1706 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1709 if (
uint64_t AndMask = MS.MapParams->AndMask)
1710 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1712 if (
uint64_t XorMask = MS.MapParams->XorMask)
1713 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1725 std::pair<Value *, Value *>
1732 assert(VectTy->getElementType()->isPointerTy());
1734 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1735 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1736 Value *ShadowLong = ShadowOffset;
1737 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1739 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1742 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1744 Value *OriginPtr =
nullptr;
1745 if (MS.TrackOrigins) {
1746 Value *OriginLong = ShadowOffset;
1747 uint64_t OriginBase = MS.MapParams->OriginBase;
1748 if (OriginBase != 0)
1750 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1753 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1756 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1758 return std::make_pair(ShadowPtr, OriginPtr);
1761 template <
typename... ArgsTy>
1766 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1767 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1770 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1773 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1777 Value *ShadowOriginPtrs;
1785 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1787 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1788 ShadowOriginPtrs = createMetadataCall(
1790 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1797 return std::make_pair(ShadowPtr, OriginPtr);
1803 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1810 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1814 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1815 Value *ShadowPtrs = ConstantInt::getNullValue(
1817 Value *OriginPtrs =
nullptr;
1818 if (MS.TrackOrigins)
1819 OriginPtrs = ConstantInt::getNullValue(
1821 for (
unsigned i = 0; i < NumElements; ++i) {
1824 auto [ShadowPtr, OriginPtr] =
1825 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1828 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1829 if (MS.TrackOrigins)
1831 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1833 return {ShadowPtrs, OriginPtrs};
1840 if (MS.CompileKernel)
1841 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1842 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1857 if (!MS.TrackOrigins)
1871 Value *getOriginPtrForRetval() {
1873 return MS.RetvalOriginTLS;
1878 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1879 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1884 if (!MS.TrackOrigins)
1886 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1887 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1888 OriginMap[
V] = Origin;
1892 Type *ShadowTy = getShadowTy(OrigTy);
1902 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1907 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1909 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1911 getPoisonedShadow(AT->getElementType()));
1914 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1916 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1917 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1925 Type *ShadowTy = getShadowTy(V);
1928 return getPoisonedShadow(ShadowTy);
1940 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1941 return getCleanShadow(V);
1943 Value *Shadow = ShadowMap[
V];
1945 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1947 assert(Shadow &&
"No shadow for a value");
1951 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1952 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1953 : getCleanShadow(V);
1958 if (
Argument *
A = dyn_cast<Argument>(V)) {
1960 Value *&ShadowPtr = ShadowMap[
V];
1965 unsigned ArgOffset = 0;
1967 for (
auto &FArg :
F->args()) {
1968 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1970 ?
"vscale not fully supported\n"
1971 :
"Arg is not sized\n"));
1973 ShadowPtr = getCleanShadow(V);
1974 setOrigin(
A, getCleanOrigin());
1980 unsigned Size = FArg.hasByValAttr()
1981 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1982 :
DL.getTypeAllocSize(FArg.getType());
1986 if (FArg.hasByValAttr()) {
1990 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1991 FArg.getParamAlign(), FArg.getParamByValType());
1992 Value *CpShadowPtr, *CpOriginPtr;
1993 std::tie(CpShadowPtr, CpOriginPtr) =
1994 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1996 if (!PropagateShadow || Overflow) {
1998 EntryIRB.CreateMemSet(
2002 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2004 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
2009 if (MS.TrackOrigins) {
2011 getOriginPtrForArgument(EntryIRB, ArgOffset);
2015 EntryIRB.CreateMemCpy(
2024 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2025 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2026 ShadowPtr = getCleanShadow(V);
2027 setOrigin(
A, getCleanOrigin());
2030 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2031 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2033 if (MS.TrackOrigins) {
2035 getOriginPtrForArgument(EntryIRB, ArgOffset);
2036 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2040 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2046 assert(ShadowPtr &&
"Could not find shadow for an argument");
2050 return getCleanShadow(V);
2055 return getShadow(
I->getOperand(i));
2060 if (!MS.TrackOrigins)
2062 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2063 return getCleanOrigin();
2064 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2065 "Unexpected value type in getOrigin()");
2067 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2068 return getCleanOrigin();
2070 Value *Origin = OriginMap[
V];
2071 assert(Origin &&
"Missing origin");
2077 return getOrigin(
I->getOperand(i));
2090 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2091 << *OrigIns <<
"\n");
2096 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2097 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2098 "Can only insert checks for integer, vector, and aggregate shadow "
2101 InstrumentationList.push_back(
2102 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2111 Value *Shadow, *Origin;
2113 Shadow = getShadow(Val);
2116 Origin = getOrigin(Val);
2118 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2121 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2123 insertShadowCheck(Shadow, Origin, OrigIns);
2128 case AtomicOrdering::NotAtomic:
2129 return AtomicOrdering::NotAtomic;
2130 case AtomicOrdering::Unordered:
2131 case AtomicOrdering::Monotonic:
2132 case AtomicOrdering::Release:
2133 return AtomicOrdering::Release;
2134 case AtomicOrdering::Acquire:
2135 case AtomicOrdering::AcquireRelease:
2136 return AtomicOrdering::AcquireRelease;
2137 case AtomicOrdering::SequentiallyConsistent:
2138 return AtomicOrdering::SequentiallyConsistent;
2144 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2145 uint32_t OrderingTable[NumOrderings] = {};
2147 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2148 OrderingTable[(
int)AtomicOrderingCABI::release] =
2149 (int)AtomicOrderingCABI::release;
2150 OrderingTable[(int)AtomicOrderingCABI::consume] =
2151 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2152 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2153 (
int)AtomicOrderingCABI::acq_rel;
2154 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2155 (
int)AtomicOrderingCABI::seq_cst;
2162 case AtomicOrdering::NotAtomic:
2163 return AtomicOrdering::NotAtomic;
2164 case AtomicOrdering::Unordered:
2165 case AtomicOrdering::Monotonic:
2166 case AtomicOrdering::Acquire:
2167 return AtomicOrdering::Acquire;
2168 case AtomicOrdering::Release:
2169 case AtomicOrdering::AcquireRelease:
2170 return AtomicOrdering::AcquireRelease;
2171 case AtomicOrdering::SequentiallyConsistent:
2172 return AtomicOrdering::SequentiallyConsistent;
2178 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2179 uint32_t OrderingTable[NumOrderings] = {};
2181 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2182 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2183 OrderingTable[(int)AtomicOrderingCABI::consume] =
2184 (
int)AtomicOrderingCABI::acquire;
2185 OrderingTable[(int)AtomicOrderingCABI::release] =
2186 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2187 (int)AtomicOrderingCABI::acq_rel;
2188 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2189 (
int)AtomicOrderingCABI::seq_cst;
2197 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2200 if (isInPrologue(
I))
2205 setShadow(&
I, getCleanShadow(&
I));
2206 setOrigin(&
I, getCleanOrigin());
2218 assert(
I.getType()->isSized() &&
"Load type must have size");
2219 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2220 NextNodeIRBuilder IRB(&
I);
2221 Type *ShadowTy = getShadowTy(&
I);
2223 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2224 const Align Alignment =
I.getAlign();
2225 if (PropagateShadow) {
2226 std::tie(ShadowPtr, OriginPtr) =
2227 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2231 setShadow(&
I, getCleanShadow(&
I));
2235 insertShadowCheck(
I.getPointerOperand(), &
I);
2240 if (MS.TrackOrigins) {
2241 if (PropagateShadow) {
2246 setOrigin(&
I, getCleanOrigin());
2256 StoreList.push_back(&
I);
2258 insertShadowCheck(
I.getPointerOperand(), &
I);
2262 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2266 Value *Val =
I.getOperand(1);
2267 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2272 insertShadowCheck(
Addr, &
I);
2277 if (isa<AtomicCmpXchgInst>(
I))
2278 insertShadowCheck(Val, &
I);
2282 setShadow(&
I, getCleanShadow(&
I));
2283 setOrigin(&
I, getCleanOrigin());
2298 insertShadowCheck(
I.getOperand(1), &
I);
2302 setOrigin(&
I, getOrigin(&
I, 0));
2306 insertShadowCheck(
I.getOperand(2), &
I);
2308 auto *Shadow0 = getShadow(&
I, 0);
2309 auto *Shadow1 = getShadow(&
I, 1);
2312 setOriginForNaryOp(
I);
2317 auto *Shadow0 = getShadow(&
I, 0);
2318 auto *Shadow1 = getShadow(&
I, 1);
2321 setOriginForNaryOp(
I);
2327 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2328 setOrigin(&
I, getOrigin(&
I, 0));
2333 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2334 setOrigin(&
I, getOrigin(&
I, 0));
2339 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2340 setOrigin(&
I, getOrigin(&
I, 0));
2347 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2348 if (CI->isMustTailCall())
2352 setOrigin(&
I, getOrigin(&
I, 0));
2358 "_msprop_ptrtoint"));
2359 setOrigin(&
I, getOrigin(&
I, 0));
2365 "_msprop_inttoptr"));
2366 setOrigin(&
I, getOrigin(&
I, 0));
2369 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2370 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2371 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2372 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2373 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2374 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2389 Value *S2 = getShadow(&
I, 1);
2390 Value *V1 =
I.getOperand(0);
2399 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2400 setOriginForNaryOp(
I);
2411 Value *S2 = getShadow(&
I, 1);
2421 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2422 setOriginForNaryOp(
I);
2440 template <
bool CombineShadow>
class Combiner {
2441 Value *Shadow =
nullptr;
2442 Value *Origin =
nullptr;
2444 MemorySanitizerVisitor *MSV;
2448 : IRB(IRB), MSV(MSV) {}
2452 if (CombineShadow) {
2457 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2458 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2462 if (MSV->MS.TrackOrigins) {
2467 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2469 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2470 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2480 Value *OpShadow = MSV->getShadow(V);
2481 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2482 return Add(OpShadow, OpOrigin);
2488 if (CombineShadow) {
2490 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2491 MSV->setShadow(
I, Shadow);
2493 if (MSV->MS.TrackOrigins) {
2495 MSV->setOrigin(
I, Origin);
2505 if (!MS.TrackOrigins)
2508 OriginCombiner
OC(
this, IRB);
2509 for (
Use &
Op :
I.operands())
2514 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2516 "Vector of pointers is not a valid shadow type");
2517 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2526 Type *srcTy =
V->getType();
2529 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2530 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2531 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2537 cast<VectorType>(dstTy)->getElementCount() ==
2538 cast<VectorType>(srcTy)->getElementCount())
2549 Type *ShadowTy = getShadowTy(V);
2550 if (
V->getType() == ShadowTy)
2552 if (
V->getType()->isPtrOrPtrVectorTy())
2561 ShadowAndOriginCombiner
SC(
this, IRB);
2562 for (
Use &
Op :
I.operands())
2582 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2583 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2584 Type *EltTy = VTy->getElementType();
2586 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2589 const APInt &
V = Elt->getValue();
2591 Elements.push_back(ConstantInt::get(EltTy, V2));
2593 Elements.push_back(ConstantInt::get(EltTy, 1));
2598 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2599 const APInt &
V = Elt->getValue();
2601 ShadowMul = ConstantInt::get(Ty, V2);
2603 ShadowMul = ConstantInt::get(Ty, 1);
2609 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2610 setOrigin(&
I, getOrigin(OtherArg));
2614 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2615 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2616 if (constOp0 && !constOp1)
2617 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2618 else if (constOp1 && !constOp0)
2619 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2634 insertShadowCheck(
I.getOperand(1), &
I);
2635 setShadow(&
I, getShadow(&
I, 0));
2636 setOrigin(&
I, getOrigin(&
I, 0));
2653 void handleEqualityComparison(
ICmpInst &
I) {
2657 Value *Sa = getShadow(
A);
2658 Value *Sb = getShadow(
B);
2684 setOriginForNaryOp(
I);
2726 void handleRelationalComparisonExact(
ICmpInst &
I) {
2730 Value *Sa = getShadow(
A);
2731 Value *Sb = getShadow(
B);
2742 bool IsSigned =
I.isSigned();
2744 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2745 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2747 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2748 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2751 setOriginForNaryOp(
I);
2758 void handleSignedRelationalComparison(
ICmpInst &
I) {
2762 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2763 op =
I.getOperand(0);
2764 pre =
I.getPredicate();
2765 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2766 op =
I.getOperand(1);
2767 pre =
I.getSwappedPredicate();
2780 setShadow(&
I, Shadow);
2781 setOrigin(&
I, getOrigin(
op));
2792 if (
I.isEquality()) {
2793 handleEqualityComparison(
I);
2799 handleRelationalComparisonExact(
I);
2803 handleSignedRelationalComparison(
I);
2808 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2809 handleRelationalComparisonExact(
I);
2816 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2823 Value *S2 = getShadow(&
I, 1);
2828 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2829 setOriginForNaryOp(
I);
2840 Value *S0 = getShadow(&
I, 0);
2842 Value *S2 = getShadow(&
I, 2);
2847 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2849 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2850 setOriginForNaryOp(
I);
2864 getShadow(
I.getArgOperand(1));
2867 {I.getArgOperand(0), I.getArgOperand(1),
2868 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2869 I.eraseFromParent();
2887 getShadow(
I.getArgOperand(1));
2890 {I.getArgOperand(0), I.getArgOperand(1),
2891 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2892 I.eraseFromParent();
2900 {I.getArgOperand(0),
2901 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2902 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2903 I.eraseFromParent();
2906 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2908 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2917 Value *Shadow = getShadow(&
I, 1);
2918 Value *ShadowPtr, *OriginPtr;
2922 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2927 insertShadowCheck(
Addr, &
I);
2930 if (MS.TrackOrigins)
2943 Type *ShadowTy = getShadowTy(&
I);
2944 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2945 if (PropagateShadow) {
2949 std::tie(ShadowPtr, OriginPtr) =
2950 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2954 setShadow(&
I, getCleanShadow(&
I));
2958 insertShadowCheck(
Addr, &
I);
2960 if (MS.TrackOrigins) {
2961 if (PropagateShadow)
2962 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2964 setOrigin(&
I, getCleanOrigin());
2977 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2978 RetTy->isX86_MMXTy()))
2981 unsigned NumArgOperands =
I.arg_size();
2982 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2983 Type *Ty =
I.getArgOperand(i)->getType();
2989 ShadowAndOriginCombiner
SC(
this, IRB);
2990 for (
unsigned i = 0; i < NumArgOperands; ++i)
2991 SC.Add(
I.getArgOperand(i));
3008 unsigned NumArgOperands =
I.arg_size();
3009 if (NumArgOperands == 0)
3012 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3013 I.getArgOperand(1)->getType()->isVectorTy() &&
3014 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
3016 return handleVectorStoreIntrinsic(
I);
3019 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3020 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
3022 return handleVectorLoadIntrinsic(
I);
3025 if (
I.doesNotAccessMemory())
3026 if (maybeHandleSimpleNomemIntrinsic(
I))
3034 setShadow(&
I, getShadow(&
I, 0));
3035 setOrigin(&
I, getOrigin(&
I, 0));
3043 InstrumentLifetimeStart =
false;
3044 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3050 Type *OpType =
Op->getType();
3052 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3054 setOrigin(&
I, getOrigin(
Op));
3059 Value *Src =
I.getArgOperand(0);
3065 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3068 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3071 Value *OutputShadow =
3072 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3074 setShadow(&
I, OutputShadow);
3075 setOriginForNaryOp(
I);
3093 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3094 bool HasRoundingMode =
false) {
3096 Value *CopyOp, *ConvertOp;
3098 assert((!HasRoundingMode ||
3099 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3100 "Invalid rounding mode");
3102 switch (
I.arg_size() - HasRoundingMode) {
3104 CopyOp =
I.getArgOperand(0);
3105 ConvertOp =
I.getArgOperand(1);
3108 ConvertOp =
I.getArgOperand(0);
3122 Value *ConvertShadow = getShadow(ConvertOp);
3123 Value *AggShadow =
nullptr;
3126 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3127 for (
int i = 1; i < NumUsedElements; ++i) {
3129 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3130 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3133 AggShadow = ConvertShadow;
3136 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3143 Value *ResultShadow = getShadow(CopyOp);
3144 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3145 for (
int i = 0; i < NumUsedElements; ++i) {
3147 ResultShadow, ConstantInt::getNullValue(EltTy),
3150 setShadow(&
I, ResultShadow);
3151 setOrigin(&
I, getOrigin(CopyOp));
3153 setShadow(&
I, getCleanShadow(&
I));
3154 setOrigin(&
I, getCleanOrigin());
3162 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3165 return CreateShadowCast(IRB, S2,
T,
true);
3173 return CreateShadowCast(IRB, S2,
T,
true);
3190 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3196 Value *S2 = getShadow(&
I, 1);
3197 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3198 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3199 Value *V1 =
I.getOperand(0);
3202 {IRB.CreateBitCast(S1, V1->getType()), V2});
3204 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3205 setOriginForNaryOp(
I);
3209 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3210 const unsigned X86_MMXSizeInBits = 64;
3211 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3212 "Illegal MMX vector element size");
3214 X86_MMXSizeInBits / EltSizeInBits);
3221 case Intrinsic::x86_sse2_packsswb_128:
3222 case Intrinsic::x86_sse2_packuswb_128:
3223 return Intrinsic::x86_sse2_packsswb_128;
3225 case Intrinsic::x86_sse2_packssdw_128:
3226 case Intrinsic::x86_sse41_packusdw:
3227 return Intrinsic::x86_sse2_packssdw_128;
3229 case Intrinsic::x86_avx2_packsswb:
3230 case Intrinsic::x86_avx2_packuswb:
3231 return Intrinsic::x86_avx2_packsswb;
3233 case Intrinsic::x86_avx2_packssdw:
3234 case Intrinsic::x86_avx2_packusdw:
3235 return Intrinsic::x86_avx2_packssdw;
3237 case Intrinsic::x86_mmx_packsswb:
3238 case Intrinsic::x86_mmx_packuswb:
3239 return Intrinsic::x86_mmx_packsswb;
3241 case Intrinsic::x86_mmx_packssdw:
3242 return Intrinsic::x86_mmx_packssdw;
3255 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3257 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3260 Value *S2 = getShadow(&
I, 1);
3261 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3266 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3282 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3285 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3289 setOriginForNaryOp(
I);
3293 Constant *createDppMask(
unsigned Width,
unsigned Mask) {
3306 const unsigned Width =
3307 cast<FixedVectorType>(S->
getType())->getNumElements();
3313 Value *DstMaskV = createDppMask(Width, DstMask);
3333 Value *S0 = getShadow(&
I, 0);
3337 const unsigned Width =
3338 cast<FixedVectorType>(S->
getType())->getNumElements();
3339 assert(Width == 2 || Width == 4 || Width == 8);
3341 const unsigned Mask = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3342 const unsigned SrcMask =
Mask >> 4;
3343 const unsigned DstMask =
Mask & 0xf;
3346 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3351 SI1, findDppPoisonedOutput(IRB, S, SrcMask << 4, DstMask << 4));
3358 setOriginForNaryOp(
I);
3362 C = CreateAppToShadowCast(IRB,
C);
3376 Value *Sc = getShadow(&
I, 2);
3377 Value *Oc = MS.TrackOrigins ? getOrigin(
C) : nullptr;
3382 C = convertBlendvToSelectMask(IRB,
C);
3383 Sc = convertBlendvToSelectMask(IRB, Sc);
3389 handleSelectLikeInst(
I,
C,
T,
F);
3394 const unsigned SignificantBitsPerResultElement = 16;
3395 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3397 unsigned ZeroBitsPerResultElement =
3401 auto *Shadow0 = getShadow(&
I, 0);
3402 auto *Shadow1 = getShadow(&
I, 1);
3407 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3410 setOriginForNaryOp(
I);
3415 unsigned EltSizeInBits = 0) {
3416 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3417 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3419 auto *Shadow0 = getShadow(&
I, 0);
3420 auto *Shadow1 = getShadow(&
I, 1);
3427 setOriginForNaryOp(
I);
3435 Type *ResTy = getShadowTy(&
I);
3436 auto *Shadow0 = getShadow(&
I, 0);
3437 auto *Shadow1 = getShadow(&
I, 1);
3442 setOriginForNaryOp(
I);
3450 auto *Shadow0 = getShadow(&
I, 0);
3451 auto *Shadow1 = getShadow(&
I, 1);
3453 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3455 setOriginForNaryOp(
I);
3464 setOrigin(&
I, getOrigin(&
I, 0));
3472 Value *OperandShadow = getShadow(&
I, 0);
3474 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3482 setOrigin(&
I, getOrigin(&
I, 0));
3490 Value *OperandShadow = getShadow(&
I, 0);
3491 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3499 setOrigin(&
I, getOrigin(&
I, 0));
3507 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3512 insertShadowCheck(
Addr, &
I);
3523 Value *ShadowPtr, *OriginPtr;
3524 std::tie(ShadowPtr, OriginPtr) =
3525 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3528 insertShadowCheck(
Addr, &
I);
3531 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3533 insertShadowCheck(Shadow, Origin, &
I);
3540 Value *PassThru =
I.getArgOperand(2);
3543 insertShadowCheck(
Ptr, &
I);
3544 insertShadowCheck(Mask, &
I);
3547 if (!PropagateShadow) {
3548 setShadow(&
I, getCleanShadow(&
I));
3549 setOrigin(&
I, getCleanOrigin());
3553 Type *ShadowTy = getShadowTy(&
I);
3554 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3555 auto [ShadowPtr, OriginPtr] =
3556 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3559 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3561 setShadow(&
I, Shadow);
3564 setOrigin(&
I, getCleanOrigin());
3569 Value *Values =
I.getArgOperand(0);
3574 insertShadowCheck(
Ptr, &
I);
3575 insertShadowCheck(Mask, &
I);
3578 Value *Shadow = getShadow(Values);
3579 Type *ElementShadowTy =
3580 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3581 auto [ShadowPtr, OriginPtrs] =
3582 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3591 Value *Ptrs =
I.getArgOperand(0);
3592 const Align Alignment(
3593 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3595 Value *PassThru =
I.getArgOperand(3);
3597 Type *PtrsShadowTy = getShadowTy(Ptrs);
3599 insertShadowCheck(Mask, &
I);
3603 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3606 if (!PropagateShadow) {
3607 setShadow(&
I, getCleanShadow(&
I));
3608 setOrigin(&
I, getCleanOrigin());
3612 Type *ShadowTy = getShadowTy(&
I);
3613 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3614 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3615 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3619 getShadow(PassThru),
"_msmaskedgather");
3621 setShadow(&
I, Shadow);
3624 setOrigin(&
I, getCleanOrigin());
3629 Value *Values =
I.getArgOperand(0);
3630 Value *Ptrs =
I.getArgOperand(1);
3631 const Align Alignment(
3632 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3635 Type *PtrsShadowTy = getShadowTy(Ptrs);
3637 insertShadowCheck(Mask, &
I);
3641 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3644 Value *Shadow = getShadow(Values);
3645 Type *ElementShadowTy =
3646 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3647 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3648 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3657 Value *
V =
I.getArgOperand(0);
3659 const Align Alignment(
3660 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3662 Value *Shadow = getShadow(V);
3665 insertShadowCheck(
Ptr, &
I);
3666 insertShadowCheck(Mask, &
I);
3671 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3672 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3676 if (!MS.TrackOrigins)
3679 auto &
DL =
F.getDataLayout();
3680 paintOrigin(IRB, getOrigin(V), OriginPtr,
3688 const Align Alignment(
3689 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3691 Value *PassThru =
I.getArgOperand(3);
3694 insertShadowCheck(
Ptr, &
I);
3695 insertShadowCheck(Mask, &
I);
3698 if (!PropagateShadow) {
3699 setShadow(&
I, getCleanShadow(&
I));
3700 setOrigin(&
I, getCleanOrigin());
3704 Type *ShadowTy = getShadowTy(&
I);
3705 Value *ShadowPtr, *OriginPtr;
3706 std::tie(ShadowPtr, OriginPtr) =
3707 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3709 getShadow(PassThru),
"_msmaskedld"));
3711 if (!MS.TrackOrigins)
3718 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3723 setOrigin(&
I, Origin);
3733 Type *ShadowTy = getShadowTy(&
I);
3736 Value *SMask = getShadow(&
I, 1);
3741 {getShadow(&I, 0), I.getOperand(1)});
3744 setOriginForNaryOp(
I);
3749 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3766 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3767 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3768 "pclmul 3rd operand must be a constant");
3769 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3771 getPclmulMask(Width, Imm & 0x01));
3773 getPclmulMask(Width, Imm & 0x10));
3774 ShadowAndOriginCombiner SOC(
this, IRB);
3775 SOC.Add(Shuf0, getOrigin(&
I, 0));
3776 SOC.Add(Shuf1, getOrigin(&
I, 1));
3784 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3786 Value *Second = getShadow(&
I, 1);
3789 Mask.push_back(Width);
3790 for (
unsigned i = 1; i < Width; i++)
3794 setShadow(&
I, Shadow);
3795 setOriginForNaryOp(
I);
3800 Value *Shadow0 = getShadow(&
I, 0);
3801 Value *Shadow1 = getShadow(&
I, 1);
3807 setShadow(&
I, Shadow);
3808 setOriginForNaryOp(
I);
3814 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3816 Value *Second = getShadow(&
I, 1);
3820 Mask.push_back(Width);
3821 for (
unsigned i = 1; i < Width; i++)
3825 setShadow(&
I, Shadow);
3826 setOriginForNaryOp(
I);
3833 assert(
I.getType()->isIntOrIntVectorTy());
3834 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3838 setShadow(&
I, getShadow(&
I, 0));
3839 setOrigin(&
I, getOrigin(&
I, 0));
3844 Value *Shadow = getShadow(&
I, 0);
3845 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3846 setOrigin(&
I, getOrigin(&
I, 0));
3851 Value *Shadow0 = getShadow(&
I, 0);
3852 Value *Shadow1 = getShadow(&
I, 1);
3855 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3861 setShadow(&
I, Shadow);
3862 setOriginForNaryOp(
I);
3866 switch (
I.getIntrinsicID()) {
3867 case Intrinsic::uadd_with_overflow:
3868 case Intrinsic::sadd_with_overflow:
3869 case Intrinsic::usub_with_overflow:
3870 case Intrinsic::ssub_with_overflow:
3871 case Intrinsic::umul_with_overflow:
3872 case Intrinsic::smul_with_overflow:
3873 handleArithmeticWithOverflow(
I);
3875 case Intrinsic::abs:
3876 handleAbsIntrinsic(
I);
3878 case Intrinsic::is_fpclass:
3881 case Intrinsic::lifetime_start:
3882 handleLifetimeStart(
I);
3884 case Intrinsic::launder_invariant_group:
3885 case Intrinsic::strip_invariant_group:
3886 handleInvariantGroup(
I);
3888 case Intrinsic::bswap:
3891 case Intrinsic::ctlz:
3892 case Intrinsic::cttz:
3893 handleCountZeroes(
I);
3895 case Intrinsic::masked_compressstore:
3896 handleMaskedCompressStore(
I);
3898 case Intrinsic::masked_expandload:
3899 handleMaskedExpandLoad(
I);
3901 case Intrinsic::masked_gather:
3902 handleMaskedGather(
I);
3904 case Intrinsic::masked_scatter:
3905 handleMaskedScatter(
I);
3907 case Intrinsic::masked_store:
3908 handleMaskedStore(
I);
3910 case Intrinsic::masked_load:
3911 handleMaskedLoad(
I);
3913 case Intrinsic::vector_reduce_and:
3914 handleVectorReduceAndIntrinsic(
I);
3916 case Intrinsic::vector_reduce_or:
3917 handleVectorReduceOrIntrinsic(
I);
3919 case Intrinsic::vector_reduce_add:
3920 case Intrinsic::vector_reduce_xor:
3921 case Intrinsic::vector_reduce_mul:
3922 handleVectorReduceIntrinsic(
I);
3924 case Intrinsic::x86_sse_stmxcsr:
3927 case Intrinsic::x86_sse_ldmxcsr:
3930 case Intrinsic::x86_avx512_vcvtsd2usi64:
3931 case Intrinsic::x86_avx512_vcvtsd2usi32:
3932 case Intrinsic::x86_avx512_vcvtss2usi64:
3933 case Intrinsic::x86_avx512_vcvtss2usi32:
3934 case Intrinsic::x86_avx512_cvttss2usi64:
3935 case Intrinsic::x86_avx512_cvttss2usi:
3936 case Intrinsic::x86_avx512_cvttsd2usi64:
3937 case Intrinsic::x86_avx512_cvttsd2usi:
3938 case Intrinsic::x86_avx512_cvtusi2ss:
3939 case Intrinsic::x86_avx512_cvtusi642sd:
3940 case Intrinsic::x86_avx512_cvtusi642ss:
3941 handleVectorConvertIntrinsic(
I, 1,
true);
3943 case Intrinsic::x86_sse2_cvtsd2si64:
3944 case Intrinsic::x86_sse2_cvtsd2si:
3945 case Intrinsic::x86_sse2_cvtsd2ss:
3946 case Intrinsic::x86_sse2_cvttsd2si64:
3947 case Intrinsic::x86_sse2_cvttsd2si:
3948 case Intrinsic::x86_sse_cvtss2si64:
3949 case Intrinsic::x86_sse_cvtss2si:
3950 case Intrinsic::x86_sse_cvttss2si64:
3951 case Intrinsic::x86_sse_cvttss2si:
3952 handleVectorConvertIntrinsic(
I, 1);
3954 case Intrinsic::x86_sse_cvtps2pi:
3955 case Intrinsic::x86_sse_cvttps2pi:
3956 handleVectorConvertIntrinsic(
I, 2);
3959 case Intrinsic::x86_avx512_psll_w_512:
3960 case Intrinsic::x86_avx512_psll_d_512:
3961 case Intrinsic::x86_avx512_psll_q_512:
3962 case Intrinsic::x86_avx512_pslli_w_512:
3963 case Intrinsic::x86_avx512_pslli_d_512:
3964 case Intrinsic::x86_avx512_pslli_q_512:
3965 case Intrinsic::x86_avx512_psrl_w_512:
3966 case Intrinsic::x86_avx512_psrl_d_512:
3967 case Intrinsic::x86_avx512_psrl_q_512:
3968 case Intrinsic::x86_avx512_psra_w_512:
3969 case Intrinsic::x86_avx512_psra_d_512:
3970 case Intrinsic::x86_avx512_psra_q_512:
3971 case Intrinsic::x86_avx512_psrli_w_512:
3972 case Intrinsic::x86_avx512_psrli_d_512:
3973 case Intrinsic::x86_avx512_psrli_q_512:
3974 case Intrinsic::x86_avx512_psrai_w_512:
3975 case Intrinsic::x86_avx512_psrai_d_512:
3976 case Intrinsic::x86_avx512_psrai_q_512:
3977 case Intrinsic::x86_avx512_psra_q_256:
3978 case Intrinsic::x86_avx512_psra_q_128:
3979 case Intrinsic::x86_avx512_psrai_q_256:
3980 case Intrinsic::x86_avx512_psrai_q_128:
3981 case Intrinsic::x86_avx2_psll_w:
3982 case Intrinsic::x86_avx2_psll_d:
3983 case Intrinsic::x86_avx2_psll_q:
3984 case Intrinsic::x86_avx2_pslli_w:
3985 case Intrinsic::x86_avx2_pslli_d:
3986 case Intrinsic::x86_avx2_pslli_q:
3987 case Intrinsic::x86_avx2_psrl_w:
3988 case Intrinsic::x86_avx2_psrl_d:
3989 case Intrinsic::x86_avx2_psrl_q:
3990 case Intrinsic::x86_avx2_psra_w:
3991 case Intrinsic::x86_avx2_psra_d:
3992 case Intrinsic::x86_avx2_psrli_w:
3993 case Intrinsic::x86_avx2_psrli_d:
3994 case Intrinsic::x86_avx2_psrli_q:
3995 case Intrinsic::x86_avx2_psrai_w:
3996 case Intrinsic::x86_avx2_psrai_d:
3997 case Intrinsic::x86_sse2_psll_w:
3998 case Intrinsic::x86_sse2_psll_d:
3999 case Intrinsic::x86_sse2_psll_q:
4000 case Intrinsic::x86_sse2_pslli_w:
4001 case Intrinsic::x86_sse2_pslli_d:
4002 case Intrinsic::x86_sse2_pslli_q:
4003 case Intrinsic::x86_sse2_psrl_w:
4004 case Intrinsic::x86_sse2_psrl_d:
4005 case Intrinsic::x86_sse2_psrl_q:
4006 case Intrinsic::x86_sse2_psra_w:
4007 case Intrinsic::x86_sse2_psra_d:
4008 case Intrinsic::x86_sse2_psrli_w:
4009 case Intrinsic::x86_sse2_psrli_d:
4010 case Intrinsic::x86_sse2_psrli_q:
4011 case Intrinsic::x86_sse2_psrai_w:
4012 case Intrinsic::x86_sse2_psrai_d:
4013 case Intrinsic::x86_mmx_psll_w:
4014 case Intrinsic::x86_mmx_psll_d:
4015 case Intrinsic::x86_mmx_psll_q:
4016 case Intrinsic::x86_mmx_pslli_w:
4017 case Intrinsic::x86_mmx_pslli_d:
4018 case Intrinsic::x86_mmx_pslli_q:
4019 case Intrinsic::x86_mmx_psrl_w:
4020 case Intrinsic::x86_mmx_psrl_d:
4021 case Intrinsic::x86_mmx_psrl_q:
4022 case Intrinsic::x86_mmx_psra_w:
4023 case Intrinsic::x86_mmx_psra_d:
4024 case Intrinsic::x86_mmx_psrli_w:
4025 case Intrinsic::x86_mmx_psrli_d:
4026 case Intrinsic::x86_mmx_psrli_q:
4027 case Intrinsic::x86_mmx_psrai_w:
4028 case Intrinsic::x86_mmx_psrai_d:
4029 handleVectorShiftIntrinsic(
I,
false);
4031 case Intrinsic::x86_avx2_psllv_d:
4032 case Intrinsic::x86_avx2_psllv_d_256:
4033 case Intrinsic::x86_avx512_psllv_d_512:
4034 case Intrinsic::x86_avx2_psllv_q:
4035 case Intrinsic::x86_avx2_psllv_q_256:
4036 case Intrinsic::x86_avx512_psllv_q_512:
4037 case Intrinsic::x86_avx2_psrlv_d:
4038 case Intrinsic::x86_avx2_psrlv_d_256:
4039 case Intrinsic::x86_avx512_psrlv_d_512:
4040 case Intrinsic::x86_avx2_psrlv_q:
4041 case Intrinsic::x86_avx2_psrlv_q_256:
4042 case Intrinsic::x86_avx512_psrlv_q_512:
4043 case Intrinsic::x86_avx2_psrav_d:
4044 case Intrinsic::x86_avx2_psrav_d_256:
4045 case Intrinsic::x86_avx512_psrav_d_512:
4046 case Intrinsic::x86_avx512_psrav_q_128:
4047 case Intrinsic::x86_avx512_psrav_q_256:
4048 case Intrinsic::x86_avx512_psrav_q_512:
4049 handleVectorShiftIntrinsic(
I,
true);
4052 case Intrinsic::x86_sse2_packsswb_128:
4053 case Intrinsic::x86_sse2_packssdw_128:
4054 case Intrinsic::x86_sse2_packuswb_128:
4055 case Intrinsic::x86_sse41_packusdw:
4056 case Intrinsic::x86_avx2_packsswb:
4057 case Intrinsic::x86_avx2_packssdw:
4058 case Intrinsic::x86_avx2_packuswb:
4059 case Intrinsic::x86_avx2_packusdw:
4060 handleVectorPackIntrinsic(
I);
4063 case Intrinsic::x86_sse41_pblendvb:
4064 case Intrinsic::x86_sse41_blendvpd:
4065 case Intrinsic::x86_sse41_blendvps:
4066 case Intrinsic::x86_avx_blendv_pd_256:
4067 case Intrinsic::x86_avx_blendv_ps_256:
4068 case Intrinsic::x86_avx2_pblendvb:
4069 handleBlendvIntrinsic(
I);
4072 case Intrinsic::x86_avx_dp_ps_256:
4073 case Intrinsic::x86_sse41_dppd:
4074 case Intrinsic::x86_sse41_dpps:
4075 handleDppIntrinsic(
I);
4078 case Intrinsic::x86_mmx_packsswb:
4079 case Intrinsic::x86_mmx_packuswb:
4080 handleVectorPackIntrinsic(
I, 16);
4083 case Intrinsic::x86_mmx_packssdw:
4084 handleVectorPackIntrinsic(
I, 32);
4087 case Intrinsic::x86_mmx_psad_bw:
4088 case Intrinsic::x86_sse2_psad_bw:
4089 case Intrinsic::x86_avx2_psad_bw:
4090 handleVectorSadIntrinsic(
I);
4093 case Intrinsic::x86_sse2_pmadd_wd:
4094 case Intrinsic::x86_avx2_pmadd_wd:
4095 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
4096 case Intrinsic::x86_avx2_pmadd_ub_sw:
4097 handleVectorPmaddIntrinsic(
I);
4100 case Intrinsic::x86_ssse3_pmadd_ub_sw:
4101 handleVectorPmaddIntrinsic(
I, 8);
4104 case Intrinsic::x86_mmx_pmadd_wd:
4105 handleVectorPmaddIntrinsic(
I, 16);
4108 case Intrinsic::x86_sse_cmp_ss:
4109 case Intrinsic::x86_sse2_cmp_sd:
4110 case Intrinsic::x86_sse_comieq_ss:
4111 case Intrinsic::x86_sse_comilt_ss:
4112 case Intrinsic::x86_sse_comile_ss:
4113 case Intrinsic::x86_sse_comigt_ss:
4114 case Intrinsic::x86_sse_comige_ss:
4115 case Intrinsic::x86_sse_comineq_ss:
4116 case Intrinsic::x86_sse_ucomieq_ss:
4117 case Intrinsic::x86_sse_ucomilt_ss:
4118 case Intrinsic::x86_sse_ucomile_ss:
4119 case Intrinsic::x86_sse_ucomigt_ss:
4120 case Intrinsic::x86_sse_ucomige_ss:
4121 case Intrinsic::x86_sse_ucomineq_ss:
4122 case Intrinsic::x86_sse2_comieq_sd:
4123 case Intrinsic::x86_sse2_comilt_sd:
4124 case Intrinsic::x86_sse2_comile_sd:
4125 case Intrinsic::x86_sse2_comigt_sd:
4126 case Intrinsic::x86_sse2_comige_sd:
4127 case Intrinsic::x86_sse2_comineq_sd:
4128 case Intrinsic::x86_sse2_ucomieq_sd:
4129 case Intrinsic::x86_sse2_ucomilt_sd:
4130 case Intrinsic::x86_sse2_ucomile_sd:
4131 case Intrinsic::x86_sse2_ucomigt_sd:
4132 case Intrinsic::x86_sse2_ucomige_sd:
4133 case Intrinsic::x86_sse2_ucomineq_sd:
4134 handleVectorCompareScalarIntrinsic(
I);
4137 case Intrinsic::x86_avx_cmp_pd_256:
4138 case Intrinsic::x86_avx_cmp_ps_256:
4139 case Intrinsic::x86_sse2_cmp_pd:
4140 case Intrinsic::x86_sse_cmp_ps:
4141 handleVectorComparePackedIntrinsic(
I);
4144 case Intrinsic::x86_bmi_bextr_32:
4145 case Intrinsic::x86_bmi_bextr_64:
4146 case Intrinsic::x86_bmi_bzhi_32:
4147 case Intrinsic::x86_bmi_bzhi_64:
4148 case Intrinsic::x86_bmi_pdep_32:
4149 case Intrinsic::x86_bmi_pdep_64:
4150 case Intrinsic::x86_bmi_pext_32:
4151 case Intrinsic::x86_bmi_pext_64:
4152 handleBmiIntrinsic(
I);
4155 case Intrinsic::x86_pclmulqdq:
4156 case Intrinsic::x86_pclmulqdq_256:
4157 case Intrinsic::x86_pclmulqdq_512:
4158 handlePclmulIntrinsic(
I);
4161 case Intrinsic::x86_sse41_round_sd:
4162 case Intrinsic::x86_sse41_round_ss:
4163 handleUnarySdSsIntrinsic(
I);
4165 case Intrinsic::x86_sse2_max_sd:
4166 case Intrinsic::x86_sse_max_ss:
4167 case Intrinsic::x86_sse2_min_sd:
4168 case Intrinsic::x86_sse_min_ss:
4169 handleBinarySdSsIntrinsic(
I);
4172 case Intrinsic::x86_avx_vtestc_pd:
4173 case Intrinsic::x86_avx_vtestc_pd_256:
4174 case Intrinsic::x86_avx_vtestc_ps:
4175 case Intrinsic::x86_avx_vtestc_ps_256:
4176 case Intrinsic::x86_avx_vtestnzc_pd:
4177 case Intrinsic::x86_avx_vtestnzc_pd_256:
4178 case Intrinsic::x86_avx_vtestnzc_ps:
4179 case Intrinsic::x86_avx_vtestnzc_ps_256:
4180 case Intrinsic::x86_avx_vtestz_pd:
4181 case Intrinsic::x86_avx_vtestz_pd_256:
4182 case Intrinsic::x86_avx_vtestz_ps:
4183 case Intrinsic::x86_avx_vtestz_ps_256:
4184 case Intrinsic::x86_avx_ptestc_256:
4185 case Intrinsic::x86_avx_ptestnzc_256:
4186 case Intrinsic::x86_avx_ptestz_256:
4187 case Intrinsic::x86_sse41_ptestc:
4188 case Intrinsic::x86_sse41_ptestnzc:
4189 case Intrinsic::x86_sse41_ptestz:
4190 handleVtestIntrinsic(
I);
4193 case Intrinsic::fshl:
4194 case Intrinsic::fshr:
4195 handleFunnelShift(
I);
4198 case Intrinsic::is_constant:
4200 setShadow(&
I, getCleanShadow(&
I));
4201 setOrigin(&
I, getCleanOrigin());
4205 if (!handleUnknownIntrinsic(
I))
4206 visitInstruction(
I);
4211 void visitLibAtomicLoad(
CallBase &CB) {
4213 assert(isa<CallInst>(CB));
4222 Value *NewOrdering =
4226 NextNodeIRBuilder NextIRB(&CB);
4227 Value *SrcShadowPtr, *SrcOriginPtr;
4228 std::tie(SrcShadowPtr, SrcOriginPtr) =
4229 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4231 Value *DstShadowPtr =
4232 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4236 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4237 if (MS.TrackOrigins) {
4238 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4240 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4241 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4245 void visitLibAtomicStore(
CallBase &CB) {
4252 Value *NewOrdering =
4256 Value *DstShadowPtr =
4274 visitAsmInstruction(CB);
4276 visitInstruction(CB);
4285 case LibFunc_atomic_load:
4286 if (!isa<CallInst>(CB)) {
4287 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4291 visitLibAtomicLoad(CB);
4293 case LibFunc_atomic_store:
4294 visitLibAtomicStore(CB);
4301 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4302 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4310 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4312 Call->removeFnAttrs(
B);
4314 Func->removeFnAttrs(
B);
4320 bool MayCheckCall = MS.EagerChecks;
4324 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4327 unsigned ArgOffset = 0;
4330 if (!
A->getType()->isSized()) {
4331 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4335 if (
A->getType()->isScalableTy()) {
4336 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is vscale: " << CB <<
"\n");
4338 insertShadowCheck(
A, &CB);
4347 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4350 insertShadowCheck(
A, &CB);
4351 Size =
DL.getTypeAllocSize(
A->getType());
4357 Value *ArgShadow = getShadow(
A);
4358 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4360 <<
" Shadow: " << *ArgShadow <<
"\n");
4364 assert(
A->getType()->isPointerTy() &&
4365 "ByVal argument is not a pointer!");
4373 Value *AShadowPtr, *AOriginPtr;
4374 std::tie(AShadowPtr, AOriginPtr) =
4375 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4377 if (!PropagateShadow) {
4384 if (MS.TrackOrigins) {
4385 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4399 Size =
DL.getTypeAllocSize(
A->getType());
4404 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4405 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4407 getOriginPtrForArgument(IRB, ArgOffset));
4411 assert(Store !=
nullptr);
4420 if (FT->isVarArg()) {
4421 VAHelper->visitCallBase(CB, IRB);
4428 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4431 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4432 setShadow(&CB, getCleanShadow(&CB));
4433 setOrigin(&CB, getCleanOrigin());
4439 Value *
Base = getShadowPtrForRetval(IRBBefore);
4440 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4443 if (isa<CallInst>(CB)) {
4447 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4452 setShadow(&CB, getCleanShadow(&CB));
4453 setOrigin(&CB, getCleanOrigin());
4460 "Could not find insertion point for retval shadow load");
4463 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4464 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4466 setShadow(&CB, RetvalShadow);
4467 if (MS.TrackOrigins)
4468 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4469 getOriginPtrForRetval()));
4473 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4474 RetVal =
I->getOperand(0);
4476 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4477 return I->isMustTailCall();
4484 Value *RetVal =
I.getReturnValue();
4490 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4491 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4492 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4495 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4497 Value *Shadow = getShadow(RetVal);
4498 bool StoreOrigin =
true;
4500 insertShadowCheck(RetVal, &
I);
4501 Shadow = getCleanShadow(RetVal);
4502 StoreOrigin =
false;
4509 if (MS.TrackOrigins && StoreOrigin)
4510 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4516 if (!PropagateShadow) {
4517 setShadow(&
I, getCleanShadow(&
I));
4518 setOrigin(&
I, getCleanOrigin());
4522 ShadowPHINodes.push_back(&
I);
4523 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4525 if (MS.TrackOrigins)
4527 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4544 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4546 Value *ShadowBase, *OriginBase;
4547 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4554 if (PoisonStack && MS.TrackOrigins) {
4555 Value *Idptr = getLocalVarIdptr(
I);
4557 Value *Descr = getLocalVarDescription(
I);
4558 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4559 {&I, Len, Idptr, Descr});
4561 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4567 Value *Descr = getLocalVarDescription(
I);
4569 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4571 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4578 NextNodeIRBuilder IRB(InsPoint);
4580 TypeSize TS =
DL.getTypeAllocSize(
I.getAllocatedType());
4582 if (
I.isArrayAllocation())
4586 if (MS.CompileKernel)
4587 poisonAllocaKmsan(
I, IRB, Len);
4589 poisonAllocaUserspace(
I, IRB, Len);
4593 setShadow(&
I, getCleanShadow(&
I));
4594 setOrigin(&
I, getCleanOrigin());
4606 handleSelectLikeInst(
I,
B,
C,
D);
4612 Value *Sb = getShadow(
B);
4613 Value *Sc = getShadow(
C);
4614 Value *Sd = getShadow(
D);
4616 Value *Ob = MS.TrackOrigins ? getOrigin(
B) : nullptr;
4617 Value *Oc = MS.TrackOrigins ? getOrigin(
C) : nullptr;
4618 Value *Od = MS.TrackOrigins ? getOrigin(
D) : nullptr;
4623 if (
I.getType()->isAggregateType()) {
4627 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4635 C = CreateAppToShadowCast(IRB,
C);
4636 D = CreateAppToShadowCast(IRB,
D);
4643 if (MS.TrackOrigins) {
4646 if (
B->getType()->isVectorTy()) {
4647 B = convertToBool(
B, IRB);
4648 Sb = convertToBool(Sb, IRB);
4659 setShadow(&
I, getCleanShadow(&
I));
4660 setOrigin(&
I, getCleanOrigin());
4664 setShadow(&
I, getCleanShadow(&
I));
4665 setOrigin(&
I, getCleanOrigin());
4669 setShadow(&
I, getCleanShadow(&
I));
4670 setOrigin(&
I, getCleanOrigin());
4677 Value *Agg =
I.getAggregateOperand();
4679 Value *AggShadow = getShadow(Agg);
4683 setShadow(&
I, ResShadow);
4684 setOriginForNaryOp(
I);
4690 Value *AggShadow = getShadow(
I.getAggregateOperand());
4691 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4697 setOriginForNaryOp(
I);
4701 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4702 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4704 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4706 errs() <<
"QQQ " <<
I <<
"\n";
4733 insertShadowCheck(Operand, &
I);
4740 auto Size =
DL.getTypeStoreSize(ElemTy);
4742 if (MS.CompileKernel) {
4743 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4749 auto [ShadowPtr,
_] =
4750 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4761 int NumRetOutputs = 0;
4763 Type *
RetTy = cast<Value>(CB)->getType();
4764 if (!
RetTy->isVoidTy()) {
4766 auto *
ST = dyn_cast<StructType>(
RetTy);
4768 NumRetOutputs =
ST->getNumElements();
4774 switch (
Info.Type) {
4782 return NumOutputs - NumRetOutputs;
4805 int OutputArgs = getNumOutputArgs(IA, CB);
4811 for (
int i = OutputArgs; i < NumOperands; i++) {
4819 for (
int i = 0; i < OutputArgs; i++) {
4825 setShadow(&
I, getCleanShadow(&
I));
4826 setOrigin(&
I, getCleanOrigin());
4831 setShadow(&
I, getCleanShadow(&
I));
4832 setOrigin(&
I, getCleanOrigin());
4840 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4841 Value *Operand =
I.getOperand(i);
4843 insertShadowCheck(Operand, &
I);
4845 setShadow(&
I, getCleanShadow(&
I));
4846 setOrigin(&
I, getCleanOrigin());
4850struct VarArgHelperBase :
public VarArgHelper {
4852 MemorySanitizer &MS;
4853 MemorySanitizerVisitor &MSV;
4855 const unsigned VAListTagSize;
4857 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4858 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4859 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4863 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4868 unsigned ArgOffset) {
4877 unsigned ArgOffset,
unsigned ArgSize) {
4881 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4896 unsigned BaseOffset) {
4905 TailSize,
Align(8));
4910 Value *VAListTag =
I.getArgOperand(0);
4912 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4913 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4916 VAListTagSize, Alignment,
false);
4923 unpoisonVAListTagForInst(
I);
4929 unpoisonVAListTagForInst(
I);
4934struct VarArgAMD64Helper :
public VarArgHelperBase {
4937 static const unsigned AMD64GpEndOffset = 48;
4938 static const unsigned AMD64FpEndOffsetSSE = 176;
4940 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4942 unsigned AMD64FpEndOffset;
4945 Value *VAArgOverflowSize =
nullptr;
4947 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4949 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4950 MemorySanitizerVisitor &MSV)
4951 : VarArgHelperBase(
F, MS, MSV, 24) {
4952 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4953 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4954 if (Attr.isStringAttribute() &&
4955 (Attr.getKindAsString() ==
"target-features")) {
4956 if (Attr.getValueAsString().contains(
"-sse"))
4957 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4963 ArgKind classifyArgument(
Value *arg) {
4966 if (
T->isX86_FP80Ty())
4968 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4969 return AK_FloatingPoint;
4970 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4971 return AK_GeneralPurpose;
4972 if (
T->isPointerTy())
4973 return AK_GeneralPurpose;
4986 unsigned GpOffset = 0;
4987 unsigned FpOffset = AMD64GpEndOffset;
4988 unsigned OverflowOffset = AMD64FpEndOffset;
4993 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5000 assert(
A->getType()->isPointerTy());
5002 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5004 unsigned BaseOffset = OverflowOffset;
5006 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
5007 Value *OriginBase =
nullptr;
5008 if (MS.TrackOrigins)
5009 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5010 OverflowOffset += AlignedSize;
5013 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5017 Value *ShadowPtr, *OriginPtr;
5018 std::tie(ShadowPtr, OriginPtr) =
5023 if (MS.TrackOrigins)
5027 ArgKind AK = classifyArgument(
A);
5028 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
5030 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
5032 Value *ShadowBase, *OriginBase =
nullptr;
5034 case AK_GeneralPurpose:
5035 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
5036 if (MS.TrackOrigins)
5037 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
5041 case AK_FloatingPoint:
5042 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
5043 if (MS.TrackOrigins)
5044 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5051 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5053 unsigned BaseOffset = OverflowOffset;
5055 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
5056 if (MS.TrackOrigins) {
5057 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5059 OverflowOffset += AlignedSize;
5062 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5071 Value *Shadow = MSV.getShadow(
A);
5073 if (MS.TrackOrigins) {
5074 Value *Origin = MSV.getOrigin(
A);
5076 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5082 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
5083 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5086 void finalizeInstrumentation()
override {
5087 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5088 "finalizeInstrumentation called twice");
5089 if (!VAStartInstrumentationList.
empty()) {
5096 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
5103 Intrinsic::umin, CopySize,
5107 if (MS.TrackOrigins) {
5117 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5118 CallInst *OrigInst = VAStartInstrumentationList[i];
5119 NextNodeIRBuilder IRB(OrigInst);
5122 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5125 ConstantInt::get(MS.IntptrTy, 16)),
5126 PointerType::get(RegSaveAreaPtrTy, 0));
5127 Value *RegSaveAreaPtr =
5128 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5129 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5131 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5132 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5134 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5136 if (MS.TrackOrigins)
5137 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5138 Alignment, AMD64FpEndOffset);
5139 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5142 ConstantInt::get(MS.IntptrTy, 8)),
5143 PointerType::get(OverflowArgAreaPtrTy, 0));
5144 Value *OverflowArgAreaPtr =
5145 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5146 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5147 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5148 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5152 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5154 if (MS.TrackOrigins) {
5157 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5166struct VarArgMIPS64Helper :
public VarArgHelperBase {
5168 Value *VAArgSize =
nullptr;
5170 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
5171 MemorySanitizerVisitor &MSV)
5172 : VarArgHelperBase(
F, MS, MSV, 8) {}
5175 unsigned VAArgOffset = 0;
5179 Triple TargetTriple(
F.getParent()->getTargetTriple());
5181 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5186 VAArgOffset += (8 - ArgSize);
5188 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5189 VAArgOffset += ArgSize;
5190 VAArgOffset =
alignTo(VAArgOffset, 8);
5199 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5202 void finalizeInstrumentation()
override {
5203 assert(!VAArgSize && !VAArgTLSCopy &&
5204 "finalizeInstrumentation called twice");
5208 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5210 if (!VAStartInstrumentationList.
empty()) {
5219 Intrinsic::umin, CopySize,
5227 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5228 CallInst *OrigInst = VAStartInstrumentationList[i];
5229 NextNodeIRBuilder IRB(OrigInst);
5231 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5232 Value *RegSaveAreaPtrPtr =
5234 PointerType::get(RegSaveAreaPtrTy, 0));
5235 Value *RegSaveAreaPtr =
5236 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5237 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5239 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5240 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5242 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5249struct VarArgAArch64Helper :
public VarArgHelperBase {
5250 static const unsigned kAArch64GrArgSize = 64;
5251 static const unsigned kAArch64VrArgSize = 128;
5253 static const unsigned AArch64GrBegOffset = 0;
5254 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5256 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5257 static const unsigned AArch64VrEndOffset =
5258 AArch64VrBegOffset + kAArch64VrArgSize;
5259 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5262 Value *VAArgOverflowSize =
nullptr;
5264 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5266 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5267 MemorySanitizerVisitor &MSV)
5268 : VarArgHelperBase(
F, MS, MSV, 32) {}
5271 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5272 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5273 return {AK_GeneralPurpose, 1};
5274 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5275 return {AK_FloatingPoint, 1};
5277 if (
T->isArrayTy()) {
5278 auto R = classifyArgument(
T->getArrayElementType());
5279 R.second *=
T->getScalarType()->getArrayNumElements();
5284 auto R = classifyArgument(FV->getScalarType());
5285 R.second *= FV->getNumElements();
5290 return {AK_Memory, 0};
5303 unsigned GrOffset = AArch64GrBegOffset;
5304 unsigned VrOffset = AArch64VrBegOffset;
5305 unsigned OverflowOffset = AArch64VAEndOffset;
5310 auto [AK, RegNum] = classifyArgument(
A->getType());
5311 if (AK == AK_GeneralPurpose &&
5312 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5314 if (AK == AK_FloatingPoint &&
5315 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5319 case AK_GeneralPurpose:
5320 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5321 GrOffset += 8 * RegNum;
5323 case AK_FloatingPoint:
5324 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5325 VrOffset += 16 * RegNum;
5332 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5334 unsigned BaseOffset = OverflowOffset;
5335 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5336 OverflowOffset += AlignedSize;
5339 CleanUnusedTLS(IRB,
Base, BaseOffset);
5351 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5352 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5359 ConstantInt::get(MS.IntptrTy, offset)),
5360 PointerType::get(*MS.C, 0));
5368 ConstantInt::get(MS.IntptrTy, offset)),
5369 PointerType::get(*MS.C, 0));
5371 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5374 void finalizeInstrumentation()
override {
5375 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5376 "finalizeInstrumentation called twice");
5377 if (!VAStartInstrumentationList.
empty()) {
5384 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5391 Intrinsic::umin, CopySize,
5397 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5398 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5402 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5403 CallInst *OrigInst = VAStartInstrumentationList[i];
5404 NextNodeIRBuilder IRB(OrigInst);
5423 Value *StackSaveAreaPtr =
5424 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5427 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5428 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5431 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5434 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5435 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5438 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5444 Value *GrRegSaveAreaShadowPtrOff =
5445 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5447 Value *GrRegSaveAreaShadowPtr =
5448 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5454 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5460 Value *VrRegSaveAreaShadowPtrOff =
5461 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5463 Value *VrRegSaveAreaShadowPtr =
5464 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5471 VrRegSaveAreaShadowPtrOff);
5472 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5478 Value *StackSaveAreaShadowPtr =
5479 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5484 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5487 Align(16), VAArgOverflowSize);
5493struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5495 Value *VAArgSize =
nullptr;
5497 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5498 MemorySanitizerVisitor &MSV)
5499 : VarArgHelperBase(
F, MS, MSV, 8) {}
5509 Triple TargetTriple(
F.getParent()->getTargetTriple());
5517 unsigned VAArgOffset = VAArgBase;
5521 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5523 assert(
A->getType()->isPointerTy());
5525 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5528 ArgAlign =
Align(8);
5529 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5531 Value *
Base = getShadowPtrForVAArgument(
5532 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5534 Value *AShadowPtr, *AOriginPtr;
5535 std::tie(AShadowPtr, AOriginPtr) =
5536 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5546 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5548 if (
A->getType()->isArrayTy()) {
5551 Type *ElementTy =
A->getType()->getArrayElementType();
5553 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5554 }
else if (
A->getType()->isVectorTy()) {
5556 ArgAlign =
Align(ArgSize);
5559 ArgAlign =
Align(8);
5560 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5561 if (
DL.isBigEndian()) {
5565 VAArgOffset += (8 - ArgSize);
5568 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5569 VAArgOffset - VAArgBase, ArgSize);
5573 VAArgOffset += ArgSize;
5577 VAArgBase = VAArgOffset;
5581 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5584 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5587 void finalizeInstrumentation()
override {
5588 assert(!VAArgSize && !VAArgTLSCopy &&
5589 "finalizeInstrumentation called twice");
5593 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5595 if (!VAStartInstrumentationList.
empty()) {
5605 Intrinsic::umin, CopySize,
5613 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5614 CallInst *OrigInst = VAStartInstrumentationList[i];
5615 NextNodeIRBuilder IRB(OrigInst);
5617 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5618 Value *RegSaveAreaPtrPtr =
5620 PointerType::get(RegSaveAreaPtrTy, 0));
5621 Value *RegSaveAreaPtr =
5622 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5623 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5625 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5626 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5628 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5635struct VarArgSystemZHelper :
public VarArgHelperBase {
5636 static const unsigned SystemZGpOffset = 16;
5637 static const unsigned SystemZGpEndOffset = 56;
5638 static const unsigned SystemZFpOffset = 128;
5639 static const unsigned SystemZFpEndOffset = 160;
5640 static const unsigned SystemZMaxVrArgs = 8;
5641 static const unsigned SystemZRegSaveAreaSize = 160;
5642 static const unsigned SystemZOverflowOffset = 160;
5643 static const unsigned SystemZVAListTagSize = 32;
5644 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5645 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5647 bool IsSoftFloatABI;
5650 Value *VAArgOverflowSize =
nullptr;
5652 enum class ArgKind {
5660 enum class ShadowExtension {
None,
Zero, Sign };
5662 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5663 MemorySanitizerVisitor &MSV)
5664 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5665 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5667 ArgKind classifyArgument(
Type *
T) {
5674 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5675 return ArgKind::Indirect;
5676 if (
T->isFloatingPointTy())
5677 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5678 if (
T->isIntegerTy() ||
T->isPointerTy())
5679 return ArgKind::GeneralPurpose;
5680 if (
T->isVectorTy())
5681 return ArgKind::Vector;
5682 return ArgKind::Memory;
5685 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5695 return ShadowExtension::Zero;
5699 return ShadowExtension::Sign;
5701 return ShadowExtension::None;
5705 unsigned GpOffset = SystemZGpOffset;
5706 unsigned FpOffset = SystemZFpOffset;
5707 unsigned VrIndex = 0;
5708 unsigned OverflowOffset = SystemZOverflowOffset;
5715 ArgKind AK = classifyArgument(
T);
5716 if (AK == ArgKind::Indirect) {
5717 T = PointerType::get(
T, 0);
5718 AK = ArgKind::GeneralPurpose;
5720 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5721 AK = ArgKind::Memory;
5722 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5723 AK = ArgKind::Memory;
5724 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5725 AK = ArgKind::Memory;
5726 Value *ShadowBase =
nullptr;
5727 Value *OriginBase =
nullptr;
5728 ShadowExtension SE = ShadowExtension::None;
5730 case ArgKind::GeneralPurpose: {
5735 SE = getShadowExtension(CB, ArgNo);
5737 if (SE == ShadowExtension::None) {
5739 assert(ArgAllocSize <= ArgSize);
5740 GapSize = ArgSize - ArgAllocSize;
5742 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5743 if (MS.TrackOrigins)
5744 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5746 GpOffset += ArgSize;
5752 case ArgKind::FloatingPoint: {
5761 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5762 if (MS.TrackOrigins)
5763 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5765 FpOffset += ArgSize;
5771 case ArgKind::Vector: {
5778 case ArgKind::Memory: {
5786 SE = getShadowExtension(CB, ArgNo);
5788 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5790 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5791 if (MS.TrackOrigins)
5793 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5794 OverflowOffset += ArgSize;
5801 case ArgKind::Indirect:
5804 if (ShadowBase ==
nullptr)
5806 Value *Shadow = MSV.getShadow(
A);
5807 if (SE != ShadowExtension::None)
5808 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5809 SE == ShadowExtension::Sign);
5811 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5813 if (MS.TrackOrigins) {
5814 Value *Origin = MSV.getOrigin(
A);
5816 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5820 Constant *OverflowSize = ConstantInt::get(
5821 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5822 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5826 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5830 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5831 PointerType::get(RegSaveAreaPtrTy, 0));
5832 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5833 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5835 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5836 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5841 unsigned RegSaveAreaSize =
5842 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5843 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5845 if (MS.TrackOrigins)
5846 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5847 Alignment, RegSaveAreaSize);
5853 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5857 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5858 PointerType::get(OverflowArgAreaPtrTy, 0));
5859 Value *OverflowArgAreaPtr =
5860 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5861 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5863 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5864 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5867 SystemZOverflowOffset);
5868 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5870 if (MS.TrackOrigins) {
5872 SystemZOverflowOffset);
5873 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5878 void finalizeInstrumentation()
override {
5879 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5880 "finalizeInstrumentation called twice");
5881 if (!VAStartInstrumentationList.
empty()) {
5888 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5896 Intrinsic::umin, CopySize,
5900 if (MS.TrackOrigins) {
5910 for (
size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.
size();
5911 VaStartNo < VaStartNum; VaStartNo++) {
5912 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5913 NextNodeIRBuilder IRB(OrigInst);
5915 copyRegSaveArea(IRB, VAListTag);
5916 copyOverflowArea(IRB, VAListTag);
5923using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5926struct VarArgNoOpHelper :
public VarArgHelper {
5927 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
5928 MemorySanitizerVisitor &MSV) {}
5936 void finalizeInstrumentation()
override {}
5942 MemorySanitizerVisitor &Visitor) {
5945 Triple TargetTriple(Func.getParent()->getTargetTriple());
5947 return new VarArgAMD64Helper(Func, Msan, Visitor);
5949 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5951 return new VarArgAArch64Helper(Func, Msan, Visitor);
5954 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5956 return new VarArgSystemZHelper(Func, Msan, Visitor);
5958 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5960 return new VarArgNoOpHelper(Func, Msan, Visitor);
5967 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5970 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
5974 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5977 return Visitor.runOnFunction();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DVR)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static ConstantInt * getBool(LLVMContext &Context, bool V)
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
Type * getElementType() const
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.