119 using namespace llvm;
138 "dfsan-preserve-alignment",
160 cl::desc(
"File listing native ABI functions and how the pass treats them"),
166 "dfsan-combine-pointer-labels-on-load",
167 cl::desc(
"Combine the label of the pointer with the label of the data when "
168 "loading from memory."),
174 "dfsan-combine-pointer-labels-on-store",
175 cl::desc(
"Combine the label of the pointer with the label of the data when "
176 "storing in memory."),
181 "dfsan-combine-offset-labels-on-gep",
183 "Combine the label of the offset with the label of the pointer when "
184 "doing pointer arithmetic."),
188 "dfsan-combine-taint-lookup-table",
190 "When dfsan-combine-offset-labels-on-gep and/or "
191 "dfsan-combine-pointer-labels-on-load are false, this flag can "
192 "be used to re-enable combining offset and/or pointer taint when "
193 "loading specific constant global variables (i.e. lookup tables)."),
197 "dfsan-debug-nonzero-labels",
198 cl::desc(
"Insert calls to __dfsan_nonzero_label on observing a parameter, "
199 "load or return with a nonzero label"),
213 "dfsan-event-callbacks",
214 cl::desc(
"Insert calls to __dfsan_*_callback functions on data events."),
221 "dfsan-conditional-callbacks",
227 "dfsan-track-select-control-flow",
228 cl::desc(
"Propagate labels from condition values of select instructions "
234 "dfsan-instrument-with-call-threshold",
235 cl::desc(
"If the function being instrumented requires more than "
236 "this number of origin stores, use callbacks instead of "
237 "inline checks (-1 means never use callbacks)."),
246 cl::desc(
"Track origins of labels"),
250 "dfsan-ignore-personality-routine",
251 cl::desc(
"If a personality routine is marked uninstrumented from the ABI "
252 "list, do not create a wrapper for it."),
257 Type *GType =
G.getValueType();
259 if (
StructType *SGType = dyn_cast<StructType>(GType)) {
260 if (!SGType->isLiteral())
261 return SGType->getName();
263 return "<unknown type>";
272 struct MemoryMapParams {
293 std::unique_ptr<SpecialCaseList> SCL;
296 DFSanABIList() =
default;
298 void set(std::unique_ptr<SpecialCaseList> List) { SCL =
std::move(List); }
303 return isIn(*
F.getParent(), Category) ||
304 SCL->inSection(
"dataflow",
"fun",
F.getName(), Category);
316 return SCL->inSection(
"dataflow",
"fun", GA.
getName(), Category);
318 return SCL->inSection(
"dataflow",
"global", GA.
getName(), Category) ||
325 return SCL->inSection(
"dataflow",
"src",
M.getModuleIdentifier(), Category);
332 struct TransformedFunction {
334 std::vector<unsigned> ArgumentIndexMapping)
335 : OriginalType(OriginalType), TransformedType(TransformedType),
336 ArgumentIndexMapping(ArgumentIndexMapping) {}
339 TransformedFunction(
const TransformedFunction &) =
delete;
340 TransformedFunction &operator=(
const TransformedFunction &) =
delete;
343 TransformedFunction(TransformedFunction &&) =
default;
344 TransformedFunction &operator=(TransformedFunction &&) =
default;
357 std::vector<unsigned> ArgumentIndexMapping;
364 transformFunctionAttributes(
const TransformedFunction &TransformedFunction,
368 std::vector<llvm::AttributeSet> ArgumentAttributes(
369 TransformedFunction.TransformedType->getNumParams());
374 for (
unsigned I = 0,
IE = TransformedFunction.ArgumentIndexMapping.size();
376 unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[
I];
377 ArgumentAttributes[TransformedIndex] = CallSiteAttrs.
getParamAttrs(
I);
381 for (
unsigned I = TransformedFunction.OriginalType->getNumParams(),
392 class DataFlowSanitizer {
393 friend struct DFSanFunction;
394 friend class DFSanVisitor;
396 enum { ShadowWidthBits = 8, ShadowWidthBytes = ShadowWidthBits / 8 };
398 enum { OriginWidthBits = 32, OriginWidthBytes = OriginWidthBits / 8 };
474 MDNode *OriginStoreWeights;
475 DFSanABIList ABIList;
482 const MemoryMapParams *MapParams;
487 std::pair<Value *, Value *>
491 bool isForceZeroLabels(
const Function *
F);
499 void initializeCallbackFunctions(
Module &
M);
500 void initializeRuntimeFunctions(
Module &
M);
501 void injectMetadataGlobals(
Module &
M);
502 bool initializeModule(
Module &
M);
514 bool shouldTrackOrigins();
526 bool isZeroShadow(
Value *V);
540 DataFlowSanitizer(
const std::vector<std::string> &ABIListFiles);
545 struct DFSanFunction {
546 DataFlowSanitizer &DFS;
550 bool IsForceZeroLabels;
558 struct PHIFixupElement {
563 std::vector<PHIFixupElement> PHIFixups;
566 std::vector<Value *> NonZeroChecks;
568 struct CachedShadow {
581 DFSanFunction(DataFlowSanitizer &DFS,
Function *
F,
bool IsNativeABI,
582 bool IsForceZeroLabels)
583 : DFS(DFS),
F(
F), IsNativeABI(IsNativeABI),
584 IsForceZeroLabels(IsForceZeroLabels) {
602 Value *getRetvalOriginTLS();
614 Value *combineOrigins(
const std::vector<Value *> &Shadows,
615 const std::vector<Value *> &Origins,
Instruction *Pos,
669 bool isLookupTableConstant(
Value *
P);
674 template <
class AggregateType>
675 Value *collapseAggregateShadow(AggregateType *AT,
Value *Shadow,
684 std::pair<Value *, Value *>
700 bool useCallbackLoadLabelAndOrigin(
uint64_t Size,
Align InstAlignment);
729 bool shouldInstrumentWithCall();
735 std::pair<Value *, Value *>
738 int NumOriginStores = 0;
741 class DFSanVisitor :
public InstVisitor<DFSanVisitor> {
745 DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {}
748 return DFSF.F->getParent()->getDataLayout();
758 void visitCmpInst(
CmpInst &CI);
767 void visitPHINode(
PHINode &PN);
798 DataFlowSanitizer::DataFlowSanitizer(
799 const std::vector<std::string> &ABIListFiles) {
800 std::vector<std::string> AllABIListFiles(
std::move(ABIListFiles));
807 CombineTaintLookupTableNames.insert(v);
810 TransformedFunction DataFlowSanitizer::getCustomFunctionType(
FunctionType *
T) {
817 std::vector<unsigned> ArgumentIndexMapping;
818 for (
unsigned I = 0,
E =
T->getNumParams();
I !=
E; ++
I) {
819 Type *ParamType =
T->getParamType(
I);
820 ArgumentIndexMapping.push_back(ArgTypes.size());
821 ArgTypes.push_back(ParamType);
823 for (
unsigned I = 0,
E =
T->getNumParams();
I !=
E; ++
I)
824 ArgTypes.push_back(PrimitiveShadowTy);
826 ArgTypes.push_back(PrimitiveShadowPtrTy);
827 Type *RetType =
T->getReturnType();
829 ArgTypes.push_back(PrimitiveShadowPtrTy);
831 if (shouldTrackOrigins()) {
832 for (
unsigned I = 0,
E =
T->getNumParams();
I !=
E; ++
I)
833 ArgTypes.push_back(OriginTy);
835 ArgTypes.push_back(OriginPtrTy);
837 ArgTypes.push_back(OriginPtrTy);
840 return TransformedFunction(
842 ArgumentIndexMapping);
845 bool DataFlowSanitizer::isZeroShadow(
Value *V) {
847 if (!isa<ArrayType>(
T) && !isa<StructType>(
T)) {
848 if (
const ConstantInt *CI = dyn_cast<ConstantInt>(V))
853 return isa<ConstantAggregateZero>(V);
856 bool DataFlowSanitizer::hasLoadSizeForFastPath(
uint64_t Size) {
858 return ShadowSize % 8 == 0 || ShadowSize == 4;
861 bool DataFlowSanitizer::shouldTrackOrigins() {
863 return ShouldTrackOrigins;
866 Constant *DataFlowSanitizer::getZeroShadow(
Type *OrigTy) {
867 if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
868 return ZeroPrimitiveShadow;
869 Type *ShadowTy = getShadowTy(OrigTy);
874 return getZeroShadow(V->
getType());
880 if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
883 if (
ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
884 for (
unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
885 Indices.push_back(Idx);
887 Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
893 if (
StructType *
ST = dyn_cast<StructType>(SubShadowTy)) {
894 for (
unsigned Idx = 0; Idx <
ST->getNumElements(); Idx++) {
895 Indices.push_back(Idx);
897 Shadow, Indices,
ST->getElementType(Idx), PrimitiveShadow, IRB);
905 bool DFSanFunction::shouldInstrumentWithCall() {
910 Value *DFSanFunction::expandFromPrimitiveShadow(
Type *
T,
Value *PrimitiveShadow,
912 Type *ShadowTy = DFS.getShadowTy(
T);
914 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
915 return PrimitiveShadow;
917 if (DFS.isZeroShadow(PrimitiveShadow))
918 return DFS.getZeroShadow(ShadowTy);
924 PrimitiveShadow, IRB);
927 CachedCollapsedShadows[Shadow] = PrimitiveShadow;
931 template <
class AggregateType>
932 Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT,
Value *Shadow,
934 if (!AT->getNumElements())
935 return DFS.ZeroPrimitiveShadow;
938 Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
940 for (
unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
942 Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
943 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
948 Value *DFSanFunction::collapseToPrimitiveShadow(
Value *Shadow,
951 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
953 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
954 return collapseAggregateShadow<>(AT, Shadow, IRB);
956 return collapseAggregateShadow<>(
ST, Shadow, IRB);
960 Value *DFSanFunction::collapseToPrimitiveShadow(
Value *Shadow,
963 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
967 Value *&CS = CachedCollapsedShadows[Shadow];
972 Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
974 CS = PrimitiveShadow;
975 return PrimitiveShadow;
978 void DFSanFunction::addConditionalCallbacksIfEnabled(
Instruction &
I,
984 Value *CondShadow = getShadow(Condition);
985 if (DFS.shouldTrackOrigins()) {
986 Value *CondOrigin = getOrigin(Condition);
987 IRB.
CreateCall(DFS.DFSanConditionalCallbackOriginFn,
988 {CondShadow, CondOrigin});
990 IRB.
CreateCall(DFS.DFSanConditionalCallbackFn, {CondShadow});
994 Type *DataFlowSanitizer::getShadowTy(
Type *OrigTy) {
996 return PrimitiveShadowTy;
997 if (isa<IntegerType>(OrigTy))
998 return PrimitiveShadowTy;
999 if (isa<VectorType>(OrigTy))
1000 return PrimitiveShadowTy;
1001 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
1003 AT->getNumElements());
1006 for (
unsigned I = 0,
N =
ST->getNumElements();
I <
N; ++
I)
1007 Elements.push_back(getShadowTy(
ST->getElementType(
I)));
1010 return PrimitiveShadowTy;
1013 Type *DataFlowSanitizer::getShadowTy(
Value *V) {
1014 return getShadowTy(V->
getType());
1017 bool DataFlowSanitizer::initializeModule(
Module &M) {
1018 Triple TargetTriple(
M.getTargetTriple());
1028 Ctx = &
M.getContext();
1034 IntptrTy =
DL.getIntPtrType(*Ctx);
1038 Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1041 Type *DFSanLoadLabelAndOriginArgs[2] = {Int8Ptr, IntptrTy};
1042 DFSanLoadLabelAndOriginFnTy =
1047 Type *DFSanWrapperExternWeakNullArgs[2] = {Int8Ptr, Int8Ptr};
1048 DFSanWrapperExternWeakNullFnTy =
1051 Type *DFSanSetLabelArgs[4] = {PrimitiveShadowTy, OriginTy,
1054 DFSanSetLabelArgs,
false);
1055 DFSanNonzeroLabelFnTy =
1059 DFSanConditionalCallbackFnTy =
1062 Type *DFSanConditionalCallbackOriginArgs[2] = {PrimitiveShadowTy, OriginTy};
1066 DFSanCmpCallbackFnTy =
1069 DFSanChainOriginFnTy =
1071 Type *DFSanChainOriginIfTaintedArgs[2] = {PrimitiveShadowTy, OriginTy};
1073 OriginTy, DFSanChainOriginIfTaintedArgs,
false);
1075 Int8Ptr, IntptrTy, OriginTy};
1078 Type *DFSanMemOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1081 Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
1082 DFSanLoadStoreCallbackFnTy =
1085 Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1086 DFSanMemTransferCallbackFnTy =
1095 bool DataFlowSanitizer::isInstrumented(
const Function *
F) {
1096 return !ABIList.isIn(*
F,
"uninstrumented");
1099 bool DataFlowSanitizer::isInstrumented(
const GlobalAlias *GA) {
1100 return !ABIList.isIn(*GA,
"uninstrumented");
1103 bool DataFlowSanitizer::isForceZeroLabels(
const Function *
F) {
1104 return ABIList.isIn(*
F,
"force_zero_labels");
1107 DataFlowSanitizer::WrapperKind DataFlowSanitizer::getWrapperKind(
Function *
F) {
1108 if (ABIList.isIn(*
F,
"functional"))
1109 return WK_Functional;
1110 if (ABIList.isIn(*
F,
"discard"))
1112 if (ABIList.isIn(*
F,
"custom"))
1118 void DataFlowSanitizer::addGlobalNameSuffix(
GlobalValue *GV) {
1119 std::string GVName = std::string(GV->
getName()), Suffix =
".dfsan";
1128 std::string SearchStr =
".symver " + GVName +
",";
1129 size_t Pos =
Asm.find(SearchStr);
1130 if (Pos != std::string::npos) {
1131 Asm.replace(Pos, SearchStr.size(),
".symver " + GVName + Suffix +
",");
1132 Pos =
Asm.find(
"@");
1134 if (Pos == std::string::npos)
1137 Asm.replace(Pos, 1, Suffix +
"@");
1142 void DataFlowSanitizer::buildExternWeakCheckIfNeeded(
IRBuilder<> &IRB,
1152 std::vector<Value *>
Args;
1165 NewFName,
F->getParent());
1171 if (
F->isVarArg()) {
1192 void DataFlowSanitizer::initializeRuntimeFunctions(
Module &M) {
1195 AL =
AL.addFnAttribute(
M.getContext(), Attribute::NoUnwind);
1196 AL =
AL.addFnAttribute(
M.getContext(), Attribute::ReadOnly);
1197 AL =
AL.addRetAttribute(
M.getContext(), Attribute::ZExt);
1203 AL =
AL.addFnAttribute(
M.getContext(), Attribute::NoUnwind);
1204 AL =
AL.addFnAttribute(
M.getContext(), Attribute::ReadOnly);
1205 AL =
AL.addRetAttribute(
M.getContext(), Attribute::ZExt);
1207 "__dfsan_load_label_and_origin", DFSanLoadLabelAndOriginFnTy,
AL);
1209 DFSanUnimplementedFn =
1212 "__dfsan_wrapper_extern_weak_null", DFSanWrapperExternWeakNullFnTy);
1215 AL =
AL.addParamAttribute(
M.getContext(), 0, Attribute::ZExt);
1216 AL =
AL.addParamAttribute(
M.getContext(), 1, Attribute::ZExt);
1220 DFSanNonzeroLabelFn =
1223 DFSanVarargWrapperFnTy);
1226 AL =
AL.addParamAttribute(
M.getContext(), 0, Attribute::ZExt);
1227 AL =
AL.addRetAttribute(
M.getContext(), Attribute::ZExt);
1229 DFSanChainOriginFnTy,
AL);
1233 AL =
AL.addParamAttribute(
M.getContext(), 0, Attribute::ZExt);
1234 AL =
AL.addParamAttribute(
M.getContext(), 1, Attribute::ZExt);
1235 AL =
AL.addRetAttribute(
M.getContext(), Attribute::ZExt);
1237 "__dfsan_chain_origin_if_tainted", DFSanChainOriginIfTaintedFnTy,
AL);
1240 "__dfsan_mem_origin_transfer", DFSanMemOriginTransferFnTy);
1244 AL =
AL.addParamAttribute(
M.getContext(), 0, Attribute::ZExt);
1245 AL =
AL.addParamAttribute(
M.getContext(), 3, Attribute::ZExt);
1247 "__dfsan_maybe_store_origin", DFSanMaybeStoreOriginFnTy,
AL);
1250 DFSanRuntimeFunctions.insert(
1251 DFSanUnionLoadFn.getCallee()->stripPointerCasts());
1252 DFSanRuntimeFunctions.insert(
1253 DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
1254 DFSanRuntimeFunctions.insert(
1255 DFSanUnimplementedFn.getCallee()->stripPointerCasts());
1256 DFSanRuntimeFunctions.insert(
1257 DFSanWrapperExternWeakNullFn.getCallee()->stripPointerCasts());
1258 DFSanRuntimeFunctions.insert(
1259 DFSanSetLabelFn.getCallee()->stripPointerCasts());
1260 DFSanRuntimeFunctions.insert(
1261 DFSanNonzeroLabelFn.getCallee()->stripPointerCasts());
1262 DFSanRuntimeFunctions.insert(
1263 DFSanVarargWrapperFn.getCallee()->stripPointerCasts());
1264 DFSanRuntimeFunctions.insert(
1265 DFSanLoadCallbackFn.getCallee()->stripPointerCasts());
1266 DFSanRuntimeFunctions.insert(
1267 DFSanStoreCallbackFn.getCallee()->stripPointerCasts());
1268 DFSanRuntimeFunctions.insert(
1269 DFSanMemTransferCallbackFn.getCallee()->stripPointerCasts());
1270 DFSanRuntimeFunctions.insert(
1271 DFSanConditionalCallbackFn.getCallee()->stripPointerCasts());
1272 DFSanRuntimeFunctions.insert(
1273 DFSanConditionalCallbackOriginFn.getCallee()->stripPointerCasts());
1274 DFSanRuntimeFunctions.insert(
1275 DFSanCmpCallbackFn.getCallee()->stripPointerCasts());
1276 DFSanRuntimeFunctions.insert(
1277 DFSanChainOriginFn.getCallee()->stripPointerCasts());
1278 DFSanRuntimeFunctions.insert(
1279 DFSanChainOriginIfTaintedFn.getCallee()->stripPointerCasts());
1280 DFSanRuntimeFunctions.insert(
1281 DFSanMemOriginTransferFn.getCallee()->stripPointerCasts());
1282 DFSanRuntimeFunctions.insert(
1283 DFSanMaybeStoreOriginFn.getCallee()->stripPointerCasts());
1287 void DataFlowSanitizer::initializeCallbackFunctions(
Module &M) {
1289 DFSanLoadStoreCallbackFnTy);
1291 DFSanLoadStoreCallbackFnTy);
1293 "__dfsan_mem_transfer_callback", DFSanMemTransferCallbackFnTy);
1294 DFSanCmpCallbackFn =
1298 "__dfsan_conditional_callback", DFSanConditionalCallbackFnTy);
1299 DFSanConditionalCallbackOriginFn =
1301 DFSanConditionalCallbackOriginFnTy);
1304 void DataFlowSanitizer::injectMetadataGlobals(
Module &M) {
1320 "__dfsan_shadow_width_bytes");
1325 initializeModule(
M);
1327 if (ABIList.isIn(
M,
"skip"))
1330 const unsigned InitialGlobalSize =
M.global_size();
1331 const unsigned InitialModuleSize =
M.size();
1333 bool Changed =
false;
1339 Changed |=
G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
1340 G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
1347 GetOrInsertGlobal(
"__dfsan_arg_tls",
1349 RetvalTLS = GetOrInsertGlobal(
1350 "__dfsan_retval_tls",
1352 ArgOriginTLSTy =
ArrayType::get(OriginTy, NumOfElementsInArgOrgTLS);
1353 ArgOriginTLS = GetOrInsertGlobal(
"__dfsan_arg_origin_tls", ArgOriginTLSTy);
1354 RetvalOriginTLS = GetOrInsertGlobal(
"__dfsan_retval_origin_tls", OriginTy);
1359 M, OriginTy,
true, GlobalValue::WeakODRLinkage,
1360 ConstantInt::getSigned(OriginTy,
1362 "__dfsan_track_origins");
1365 injectMetadataGlobals(
M);
1367 initializeCallbackFunctions(
M);
1368 initializeRuntimeFunctions(
M);
1370 std::vector<Function *> FnsToInstrument;
1375 if (!
F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&
F)) {
1376 FnsToInstrument.push_back(&
F);
1377 if (
F.hasPersonalityFn())
1378 PersonalityFns.
insert(
F.getPersonalityFn()->stripPointerCasts());
1382 for (
auto *
C : PersonalityFns) {
1383 assert(isa<Function>(
C) &&
"Personality routine is not a function!");
1385 if (!isInstrumented(
F))
1386 FnsToInstrument.erase(
1387 std::remove(FnsToInstrument.begin(), FnsToInstrument.end(),
F),
1388 FnsToInstrument.end());
1401 bool GAInst = isInstrumented(&GA), FInst = isInstrumented(
F);
1402 if (GAInst && FInst) {
1403 addGlobalNameSuffix(&GA);
1404 }
else if (GAInst != FInst) {
1409 buildWrapperFunction(
F,
"", GA.
getLinkage(),
F->getFunctionType());
1413 FnsToInstrument.push_back(NewF);
1417 ReadOnlyNoneAttrs.addAttribute(Attribute::ReadOnly)
1418 .addAttribute(Attribute::ReadNone);
1422 for (std::vector<Function *>::iterator FI = FnsToInstrument.begin(),
1423 FE = FnsToInstrument.end();
1431 if (isInstrumented(&
F)) {
1432 if (isForceZeroLabels(&
F))
1433 FnsWithForceZeroLabel.
insert(&
F);
1438 addGlobalNameSuffix(&
F);
1439 }
else if (!IsZeroArgsVoidRet || getWrapperKind(&
F) == WK_Custom) {
1447 F.hasLocalLinkage() ?
F.getLinkage()
1448 : GlobalValue::LinkOnceODRLinkage;
1450 Function *NewF = buildWrapperFunction(
1452 (shouldTrackOrigins() ? std::string(
"dfso$") : std::string(
"dfsw$")) +
1453 std::string(
F.getName()),
1454 WrapperLinkage, FT);
1457 Value *WrappedFnCst =
1458 ConstantExpr::getBitCast(NewF, PointerType::getUnqual(FT));
1477 auto IsNotCmpUse = [](
Use &U) ->
bool {
1478 User *Usr = U.getUser();
1481 if (
CE->getOpcode() == Instruction::ICmp) {
1486 if (
I->getOpcode() == Instruction::ICmp) {
1492 F.replaceUsesWithIf(WrappedFnCst, IsNotCmpUse);
1494 UnwrappedFnMap[WrappedFnCst] = &
F;
1497 if (!
F.isDeclaration()) {
1507 size_t N = FI - FnsToInstrument.begin(),
1508 Count = FE - FnsToInstrument.begin();
1509 FnsToInstrument.push_back(&
F);
1510 FI = FnsToInstrument.begin() +
N;
1511 FE = FnsToInstrument.begin() + Count;
1516 UnwrappedFnMap[&
F] = &
F;
1522 if (!
F ||
F->isDeclaration())
1527 DFSanFunction DFSF(*
this,
F, FnsWithNativeABI.
count(
F),
1528 FnsWithForceZeroLabel.
count(
F));
1544 if (!DFSF.SkipInsts.count(Inst))
1545 DFSanVisitor(DFSF).visit(Inst);
1556 for (DFSanFunction::PHIFixupElement &
P : DFSF.PHIFixups) {
1557 for (
unsigned Val = 0,
N =
P.Phi->getNumIncomingValues(); Val !=
N;
1559 P.ShadowPhi->setIncomingValue(
1560 Val, DFSF.getShadow(
P.Phi->getIncomingValue(Val)));
1562 P.OriginPhi->setIncomingValue(
1563 Val, DFSF.getOrigin(
P.Phi->getIncomingValue(Val)));
1572 for (
Value *V : DFSF.NonZeroChecks) {
1575 Pos =
I->getNextNode();
1577 Pos = &DFSF.F->getEntryBlock().front();
1578 while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
1581 Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
1583 IRB.
CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
1585 Ne, Pos,
false, ColdCallWeights));
1587 ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn, {});
1592 return Changed || !FnsToInstrument.empty() ||
1593 M.global_size() != InitialGlobalSize ||
M.size() != InitialModuleSize;
1609 Value *DFSanFunction::getRetvalOriginTLS() {
return DFS.RetvalOriginTLS; }
1617 assert(DFS.shouldTrackOrigins());
1618 if (!isa<Argument>(V) && !isa<Instruction>(V))
1619 return DFS.ZeroOrigin;
1620 Value *&Origin = ValOriginMap[V];
1622 if (
Argument *
A = dyn_cast<Argument>(V)) {
1624 return DFS.ZeroOrigin;
1625 if (
A->getArgNo() < DFS.NumOfElementsInArgOrgTLS) {
1626 Instruction *ArgOriginTLSPos = &*
F->getEntryBlock().begin();
1628 Value *ArgOriginPtr = getArgOriginTLS(
A->getArgNo(), IRB);
1629 Origin = IRB.
CreateLoad(DFS.OriginTy, ArgOriginPtr);
1632 Origin = DFS.ZeroOrigin;
1635 Origin = DFS.ZeroOrigin;
1642 if (!DFS.shouldTrackOrigins())
1646 ValOriginMap[
I] = Origin;
1650 unsigned ArgOffset = 0;
1652 for (
auto &FArg :
F->args()) {
1653 if (!FArg.getType()->isSized()) {
1659 unsigned Size =
DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
1672 Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
1677 return DFS.getZeroShadow(
A);
1681 if (!isa<Argument>(V) && !isa<Instruction>(V))
1682 return DFS.getZeroShadow(V);
1683 if (IsForceZeroLabels)
1684 return DFS.getZeroShadow(V);
1685 Value *&Shadow = ValShadowMap[V];
1687 if (
Argument *
A = dyn_cast<Argument>(V)) {
1689 return DFS.getZeroShadow(V);
1690 Shadow = getShadowForTLSArgument(
A);
1691 NonZeroChecks.push_back(Shadow);
1693 Shadow = DFS.getZeroShadow(V);
1701 ValShadowMap[
I] = Shadow;
1709 assert(
Addr != RetvalTLS &&
"Reinstrumenting?");
1712 uint64_t AndMask = MapParams->AndMask;
1717 uint64_t XorMask = MapParams->XorMask;
1723 std::pair<Value *, Value *>
1724 DataFlowSanitizer::getShadowOriginAddress(
Value *
Addr,
Align InstAlignment,
1728 Value *ShadowOffset = getShadowOffset(
Addr, IRB);
1729 Value *ShadowLong = ShadowOffset;
1730 uint64_t ShadowBase = MapParams->ShadowBase;
1731 if (ShadowBase != 0) {
1738 Value *OriginPtr =
nullptr;
1739 if (shouldTrackOrigins()) {
1740 Value *OriginLong = ShadowOffset;
1741 uint64_t OriginBase = MapParams->OriginBase;
1742 if (OriginBase != 0)
1754 return std::make_pair(ShadowPtr, OriginPtr);
1758 Value *ShadowOffset) {
1765 Value *ShadowOffset = getShadowOffset(
Addr, IRB);
1766 return getShadowAddress(
Addr, Pos, ShadowOffset);
1771 Value *PrimitiveValue = combineShadows(V1,
V2, Pos);
1772 return expandFromPrimitiveShadow(
T, PrimitiveValue, Pos);
1778 if (DFS.isZeroShadow(V1))
1779 return collapseToPrimitiveShadow(
V2, Pos);
1780 if (DFS.isZeroShadow(
V2))
1781 return collapseToPrimitiveShadow(V1, Pos);
1783 return collapseToPrimitiveShadow(V1, Pos);
1785 auto V1Elems = ShadowElements.
find(V1);
1786 auto V2Elems = ShadowElements.
find(
V2);
1787 if (V1Elems != ShadowElements.
end() && V2Elems != ShadowElements.
end()) {
1788 if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
1789 V2Elems->second.begin(), V2Elems->second.end())) {
1790 return collapseToPrimitiveShadow(V1, Pos);
1792 if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
1793 V1Elems->second.begin(), V1Elems->second.end())) {
1794 return collapseToPrimitiveShadow(
V2, Pos);
1796 }
else if (V1Elems != ShadowElements.
end()) {
1797 if (V1Elems->second.count(
V2))
1798 return collapseToPrimitiveShadow(V1, Pos);
1799 }
else if (V2Elems != ShadowElements.
end()) {
1800 if (V2Elems->second.count(V1))
1801 return collapseToPrimitiveShadow(
V2, Pos);
1804 auto Key = std::make_pair(V1,
V2);
1807 CachedShadow &CCS = CachedShadows[
Key];
1812 Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
1813 Value *PV2 = collapseToPrimitiveShadow(
V2, Pos);
1817 CCS.Shadow = IRB.
CreateOr(PV1, PV2);
1819 std::set<Value *> UnionElems;
1820 if (V1Elems != ShadowElements.
end()) {
1821 UnionElems = V1Elems->second;
1823 UnionElems.insert(V1);
1825 if (V2Elems != ShadowElements.
end()) {
1826 UnionElems.insert(V2Elems->second.begin(), V2Elems->second.end());
1828 UnionElems.insert(
V2);
1830 ShadowElements[CCS.Shadow] =
std::move(UnionElems);
1840 return DFS.getZeroShadow(Inst);
1844 Shadow = combineShadows(Shadow, getShadow(Inst->
getOperand(
I)), Inst);
1846 return expandFromPrimitiveShadow(Inst->
getType(), Shadow, Inst);
1850 Value *CombinedShadow = DFSF.combineOperandShadows(&
I);
1851 DFSF.setShadow(&
I, CombinedShadow);
1852 visitInstOperandOrigins(
I);
1855 Value *DFSanFunction::combineOrigins(
const std::vector<Value *> &Shadows,
1856 const std::vector<Value *> &Origins,
1858 assert(Shadows.size() == Origins.size());
1859 size_t Size = Origins.size();
1861 return DFS.ZeroOrigin;
1862 Value *Origin =
nullptr;
1864 Zero = DFS.ZeroPrimitiveShadow;
1865 for (
size_t I = 0;
I !=
Size; ++
I) {
1866 Value *OpOrigin = Origins[
I];
1867 Constant *ConstOpOrigin = dyn_cast<Constant>(OpOrigin);
1868 if (ConstOpOrigin && ConstOpOrigin->
isNullValue())
1874 Value *OpShadow = Shadows[
I];
1875 Value *PrimitiveShadow = collapseToPrimitiveShadow(OpShadow, Pos);
1880 return Origin ? Origin : DFS.ZeroOrigin;
1885 std::vector<Value *> Shadows(
Size);
1886 std::vector<Value *> Origins(
Size);
1887 for (
unsigned I = 0;
I !=
Size; ++
I) {
1891 return combineOrigins(Shadows, Origins, Inst);
1894 void DFSanVisitor::visitInstOperandOrigins(
Instruction &
I) {
1895 if (!DFSF.DFS.shouldTrackOrigins())
1897 Value *CombinedOrigin = DFSF.combineOperandOrigins(&
I);
1898 DFSF.setOrigin(&
I, CombinedOrigin);
1901 Align DFSanFunction::getShadowAlign(
Align InstAlignment) {
1906 Align DFSanFunction::getOriginAlign(
Align InstAlignment) {
1911 bool DFSanFunction::isLookupTableConstant(
Value *
P) {
1912 if (
GlobalVariable *GV = dyn_cast<GlobalVariable>(
P->stripPointerCasts()))
1913 if (GV->isConstant() && GV->
hasName())
1914 return DFS.CombineTaintLookupTableNames.count(GV->
getName());
1919 bool DFSanFunction::useCallbackLoadLabelAndOrigin(
uint64_t Size,
1920 Align InstAlignment) {
1943 Value **OriginAddr) {
1950 std::pair<Value *, Value *> DFSanFunction::loadShadowFast(
1953 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
1954 const uint64_t ShadowSize =
Size * DFS.ShadowWidthBytes;
1956 assert(
Size >= 4 &&
"Not large enough load size for fast path!");
1959 std::vector<Value *> Shadows;
1960 std::vector<Value *> Origins;
1973 Type *WideShadowTy =
1974 ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) :
Type::getInt64Ty(*DFS.Ctx);
1978 Value *CombinedWideShadow =
1982 const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
1984 auto AppendWideShadowAndOrigin = [&](
Value *WideShadow,
Value *Origin) {
1985 if (BytesPerWideShadow > 4) {
1986 assert(BytesPerWideShadow == 8);
1995 Shadows.push_back(WideShadow);
1996 Origins.push_back(DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr));
1998 Shadows.push_back(WideShadowLo);
1999 Origins.push_back(Origin);
2001 Shadows.push_back(WideShadow);
2002 Origins.push_back(Origin);
2006 if (ShouldTrackOrigins)
2007 AppendWideShadowAndOrigin(CombinedWideShadow, FirstOrigin);
2014 for (
uint64_t ByteOfs = BytesPerWideShadow; ByteOfs <
Size;
2015 ByteOfs += BytesPerWideShadow) {
2016 WideAddr = IRB.
CreateGEP(WideShadowTy, WideAddr,
2018 Value *NextWideShadow =
2020 CombinedWideShadow = IRB.
CreateOr(CombinedWideShadow, NextWideShadow);
2021 if (ShouldTrackOrigins) {
2022 Value *NextOrigin = DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr);
2023 AppendWideShadowAndOrigin(NextWideShadow, NextOrigin);
2026 for (
unsigned Width = WideShadowBitWidth / 2;
Width >= DFS.ShadowWidthBits;
2029 CombinedWideShadow = IRB.
CreateOr(CombinedWideShadow, ShrShadow);
2031 return {IRB.
CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy),
2033 ? combineOrigins(Shadows, Origins, Pos,
2038 std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
2040 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2044 const auto SI = AllocaShadowMap.
find(AI);
2045 if (
SI != AllocaShadowMap.
end()) {
2048 const auto OI = AllocaOriginMap.
find(AI);
2049 assert(!ShouldTrackOrigins || OI != AllocaOriginMap.
end());
2050 return {ShadowLI, ShouldTrackOrigins
2059 bool AllConstants =
true;
2060 for (
const Value *Obj : Objs) {
2061 if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
2063 if (isa<GlobalVariable>(Obj) && cast<GlobalVariable>(Obj)->
isConstant())
2066 AllConstants =
false;
2070 return {DFS.ZeroPrimitiveShadow,
2071 ShouldTrackOrigins ? DFS.ZeroOrigin :
nullptr};
2074 return {DFS.ZeroPrimitiveShadow,
2075 ShouldTrackOrigins ? DFS.ZeroOrigin :
nullptr};
2079 if (ShouldTrackOrigins &&
2080 useCallbackLoadLabelAndOrigin(
Size, InstAlignment)) {
2083 IRB.
CreateCall(DFS.DFSanLoadLabelAndOriginFn,
2084 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
2085 ConstantInt::get(DFS.IntptrTy, Size)});
2086 Call->addRetAttr(Attribute::ZExt);
2088 DFS.PrimitiveShadowTy),
2093 Value *ShadowAddr, *OriginAddr;
2094 std::tie(ShadowAddr, OriginAddr) =
2095 DFS.getShadowOriginAddress(
Addr, InstAlignment, Pos);
2097 const Align ShadowAlign = getShadowAlign(InstAlignment);
2098 const Align OriginAlign = getOriginAlign(InstAlignment);
2099 Value *Origin =
nullptr;
2100 if (ShouldTrackOrigins) {
2111 return {LI, Origin};
2115 Value *ShadowAddr1 = IRB.
CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
2121 return {combineShadows(
Load, Load1, Pos), Origin};
2124 bool HasSizeForFastPath = DFS.hasLoadSizeForFastPath(
Size);
2126 if (HasSizeForFastPath)
2127 return loadShadowFast(ShadowAddr, OriginAddr,
Size, ShadowAlign,
2128 OriginAlign, Origin, Pos);
2132 DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
2134 return {FallbackCall, Origin};
2137 std::pair<Value *, Value *> DFSanFunction::loadShadowOrigin(
Value *
Addr,
2139 Align InstAlignment,
2141 Value *PrimitiveShadow, *Origin;
2142 std::tie(PrimitiveShadow, Origin) =
2143 loadShadowOriginSansLoadTracking(
Addr,
Size, InstAlignment, Pos);
2144 if (DFS.shouldTrackOrigins()) {
2147 auto *ConstantShadow = dyn_cast<Constant>(PrimitiveShadow);
2148 if (!ConstantShadow || !ConstantShadow->isZeroValue())
2149 Origin = updateOriginIfTainted(PrimitiveShadow, Origin, IRB);
2152 return {PrimitiveShadow, Origin};
2157 case AtomicOrdering::NotAtomic:
2158 return AtomicOrdering::NotAtomic;
2160 case AtomicOrdering::Monotonic:
2161 case AtomicOrdering::Acquire:
2162 return AtomicOrdering::Acquire;
2163 case AtomicOrdering::Release:
2164 case AtomicOrdering::AcquireRelease:
2165 return AtomicOrdering::AcquireRelease;
2166 case AtomicOrdering::SequentiallyConsistent:
2167 return AtomicOrdering::SequentiallyConsistent;
2181 if (
auto *
GEP = dyn_cast<GEPOperator>(V)) {
2182 V =
GEP->getPointerOperand();
2184 V = cast<Operator>(V)->getOperand(0);
2187 }
else if (isa<GlobalAlias>(V)) {
2188 V = cast<GlobalAlias>(V)->getAliasee();
2190 }
while (Visited.
insert(V).second);
2195 void DFSanVisitor::visitLoadInst(
LoadInst &LI) {
2199 DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
2200 DFSF.setOrigin(&LI, DFSF.DFS.ZeroOrigin);
2213 std::vector<Value *> Shadows;
2214 std::vector<Value *> Origins;
2215 Value *PrimitiveShadow, *Origin;
2216 std::tie(PrimitiveShadow, Origin) =
2218 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2219 if (ShouldTrackOrigins) {
2220 Shadows.push_back(PrimitiveShadow);
2221 Origins.push_back(Origin);
2224 DFSF.isLookupTableConstant(
2227 PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, Pos);
2228 if (ShouldTrackOrigins) {
2229 Shadows.push_back(PtrShadow);
2233 if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
2234 DFSF.NonZeroChecks.push_back(PrimitiveShadow);
2237 DFSF.expandFromPrimitiveShadow(LI.
getType(), PrimitiveShadow, Pos);
2238 DFSF.setShadow(&LI, Shadow);
2240 if (ShouldTrackOrigins) {
2241 DFSF.setOrigin(&LI, DFSF.combineOrigins(Shadows, Origins, Pos));
2247 IRB.
CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr8});
2251 Value *DFSanFunction::updateOriginIfTainted(
Value *Shadow,
Value *Origin,
2253 assert(DFS.shouldTrackOrigins());
2254 return IRB.
CreateCall(DFS.DFSanChainOriginIfTaintedFn, {Shadow, Origin});
2258 if (!DFS.shouldTrackOrigins())
2260 return IRB.
CreateCall(DFS.DFSanChainOriginFn, V);
2264 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2266 unsigned IntptrSize =
DL.getTypeStoreSize(DFS.IntptrTy);
2267 if (IntptrSize == OriginSize)
2269 assert(IntptrSize == OriginSize * 2);
2275 Value *StoreOriginAddr,
2277 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2279 const Align IntptrAlignment =
DL.getABITypeAlign(DFS.IntptrTy);
2280 unsigned IntptrSize =
DL.getTypeStoreSize(DFS.IntptrTy);
2282 assert(IntptrSize >= OriginSize);
2286 if (Alignment >= IntptrAlignment && IntptrSize > OriginSize) {
2287 Value *IntptrOrigin = originToIntptr(IRB, Origin);
2290 for (
unsigned I = 0;
I < StoreOriginSize / IntptrSize; ++
I) {
2293 : IntptrStoreOriginPtr;
2295 Ofs += IntptrSize / OriginSize;
2296 CurrentAlignment = IntptrAlignment;
2300 for (
unsigned I = Ofs;
I < (StoreOriginSize + OriginSize - 1) / OriginSize;
2321 Value *StoreOriginAddr,
Align InstAlignment) {
2324 const Align OriginAlignment = getOriginAlign(InstAlignment);
2325 Value *CollapsedShadow = collapseToPrimitiveShadow(Shadow, Pos);
2327 if (
auto *ConstantShadow = dyn_cast<Constant>(CollapsedShadow)) {
2328 if (!ConstantShadow->isZeroValue())
2329 paintOrigin(IRB, updateOrigin(Origin, IRB), StoreOriginAddr,
Size,
2334 if (shouldInstrumentWithCall()) {
2337 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
2338 ConstantInt::get(DFS.IntptrTy, Size), Origin});
2340 Value *
Cmp = convertToBool(CollapsedShadow, IRB,
"_dfscmp");
2342 Cmp, &*IRB.
GetInsertPoint(),
false, DFS.OriginStoreWeights, &DT);
2344 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), StoreOriginAddr,
Size,
2357 Value *ShadowAddr = DFS.getShadowAddress(
Addr, Pos);
2358 Value *ExtShadowAddr =
2359 IRB.
CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowTy));
2366 Align InstAlignment,
2367 Value *PrimitiveShadow,
2370 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins() && Origin;
2373 const auto SI = AllocaShadowMap.
find(AI);
2374 if (
SI != AllocaShadowMap.
end()) {
2380 if (ShouldTrackOrigins && !DFS.isZeroShadow(PrimitiveShadow)) {
2381 const auto OI = AllocaOriginMap.
find(AI);
2382 assert(OI != AllocaOriginMap.
end() && Origin);
2389 const Align ShadowAlign = getShadowAlign(InstAlignment);
2390 if (DFS.isZeroShadow(PrimitiveShadow)) {
2391 storeZeroPrimitiveShadow(
Addr,
Size, ShadowAlign, Pos);
2396 Value *ShadowAddr, *OriginAddr;
2397 std::tie(ShadowAddr, OriginAddr) =
2398 DFS.getShadowOriginAddress(
Addr, InstAlignment, Pos);
2400 const unsigned ShadowVecSize = 8;
2401 assert(ShadowVecSize * DFS.ShadowWidthBits <= 128 &&
2402 "Shadow vector is too large!");
2406 if (LeftSize >= ShadowVecSize) {
2410 for (
unsigned I = 0;
I != ShadowVecSize; ++
I) {
2412 ShadowVec, PrimitiveShadow,
2415 Value *ShadowVecAddr =
2416 IRB.
CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowVecTy));
2418 Value *CurShadowVecAddr =
2421 LeftSize -= ShadowVecSize;
2423 }
while (LeftSize >= ShadowVecSize);
2426 while (LeftSize > 0) {
2427 Value *CurShadowAddr =
2434 if (ShouldTrackOrigins) {
2435 storeOrigin(Pos,
Addr,
Size, PrimitiveShadow, Origin, OriginAddr,
2442 case AtomicOrdering::NotAtomic:
2443 return AtomicOrdering::NotAtomic;
2445 case AtomicOrdering::Monotonic:
2446 case AtomicOrdering::Release:
2447 return AtomicOrdering::Release;
2448 case AtomicOrdering::Acquire:
2449 case AtomicOrdering::AcquireRelease:
2450 return AtomicOrdering::AcquireRelease;
2451 case AtomicOrdering::SequentiallyConsistent:
2452 return AtomicOrdering::SequentiallyConsistent;
2457 void DFSanVisitor::visitStoreInst(
StoreInst &
SI) {
2458 auto &
DL =
SI.getModule()->getDataLayout();
2459 Value *Val =
SI.getValueOperand();
2472 const bool ShouldTrackOrigins =
2473 DFSF.DFS.shouldTrackOrigins() && !
SI.isAtomic();
2474 std::vector<Value *> Shadows;
2475 std::vector<Value *> Origins;
2478 SI.isAtomic() ? DFSF.DFS.getZeroShadow(Val) : DFSF.getShadow(Val);
2480 if (ShouldTrackOrigins) {
2481 Shadows.push_back(Shadow);
2482 Origins.push_back(DFSF.getOrigin(Val));
2485 Value *PrimitiveShadow;
2487 Value *PtrShadow = DFSF.getShadow(
SI.getPointerOperand());
2488 if (ShouldTrackOrigins) {
2489 Shadows.push_back(PtrShadow);
2490 Origins.push_back(DFSF.getOrigin(
SI.getPointerOperand()));
2492 PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &
SI);
2494 PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &
SI);
2496 Value *Origin =
nullptr;
2497 if (ShouldTrackOrigins)
2498 Origin = DFSF.combineOrigins(Shadows, Origins, &
SI);
2499 DFSF.storePrimitiveShadowOrigin(
SI.getPointerOperand(),
Size,
SI.getAlign(),
2500 PrimitiveShadow, Origin, &
SI);
2504 IRB.
CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr8});
2509 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2511 Value *Val =
I.getOperand(1);
2512 const auto &
DL =
I.getModule()->getDataLayout();
2521 const Align ShadowAlign = DFSF.getShadowAlign(InstAlignment);
2522 DFSF.storeZeroPrimitiveShadow(
Addr,
Size, ShadowAlign, &
I);
2523 DFSF.setShadow(&
I, DFSF.DFS.getZeroShadow(&
I));
2524 DFSF.setOrigin(&
I, DFSF.DFS.ZeroOrigin);
2528 visitCASOrRMW(
I.getAlign(),
I);
2535 visitCASOrRMW(
I.getAlign(),
I);
2542 visitInstOperands(UO);
2546 visitInstOperands(BO);
2549 void DFSanVisitor::visitBitCastInst(
BitCastInst &BCI) {
2553 if (
auto *CI = dyn_cast<CallInst>(BCI.
getOperand(0)))
2554 if (CI->isMustTailCall())
2556 visitInstOperands(BCI);
2559 void DFSanVisitor::visitCastInst(
CastInst &CI) { visitInstOperands(CI); }
2561 void DFSanVisitor::visitCmpInst(
CmpInst &CI) {
2562 visitInstOperands(CI);
2565 Value *CombinedShadow = DFSF.getShadow(&CI);
2566 IRB.
CreateCall(DFSF.DFS.DFSanCmpCallbackFn, CombinedShadow);
2582 DFSF.setShadow(&LPI, DFSF.DFS.getZeroShadow(&LPI));
2583 DFSF.setOrigin(&LPI, DFSF.DFS.ZeroOrigin);
2588 DFSF.isLookupTableConstant(
2590 visitInstOperands(GEPI);
2597 DFSF.setShadow(&GEPI, DFSF.getShadow(BasePointer));
2598 if (DFSF.DFS.shouldTrackOrigins())
2599 DFSF.setOrigin(&GEPI, DFSF.getOrigin(BasePointer));
2603 visitInstOperands(
I);
2607 visitInstOperands(
I);
2611 visitInstOperands(
I);
2616 Value *Agg =
I.getAggregateOperand();
2617 Value *AggShadow = DFSF.getShadow(Agg);
2619 DFSF.setShadow(&
I, ResShadow);
2620 visitInstOperandOrigins(
I);
2625 Value *AggShadow = DFSF.getShadow(
I.getAggregateOperand());
2626 Value *InsShadow = DFSF.getShadow(
I.getInsertedValueOperand());
2628 DFSF.setShadow(&
I, Res);
2629 visitInstOperandOrigins(
I);
2632 void DFSanVisitor::visitAllocaInst(
AllocaInst &
I) {
2633 bool AllLoadsStores =
true;
2634 for (
User *U :
I.users()) {
2635 if (isa<LoadInst>(U))
2639 if (
SI->getPointerOperand() == &
I)
2643 AllLoadsStores =
false;
2646 if (AllLoadsStores) {
2648 DFSF.AllocaShadowMap[&
I] = IRB.
CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
2649 if (DFSF.DFS.shouldTrackOrigins()) {
2650 DFSF.AllocaOriginMap[&
I] =
2654 DFSF.setShadow(&
I, DFSF.DFS.ZeroPrimitiveShadow);
2655 DFSF.setOrigin(&
I, DFSF.DFS.ZeroOrigin);
2658 void DFSanVisitor::visitSelectInst(
SelectInst &
I) {
2659 Value *CondShadow = DFSF.getShadow(
I.getCondition());
2660 Value *TrueShadow = DFSF.getShadow(
I.getTrueValue());
2661 Value *FalseShadow = DFSF.getShadow(
I.getFalseValue());
2662 Value *ShadowSel =
nullptr;
2663 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2664 std::vector<Value *> Shadows;
2665 std::vector<Value *> Origins;
2667 ShouldTrackOrigins ? DFSF.getOrigin(
I.getTrueValue()) :
nullptr;
2668 Value *FalseOrigin =
2669 ShouldTrackOrigins ? DFSF.getOrigin(
I.getFalseValue()) :
nullptr;
2671 DFSF.addConditionalCallbacksIfEnabled(
I,
I.getCondition());
2673 if (isa<VectorType>(
I.getCondition()->getType())) {
2674 ShadowSel = DFSF.combineShadowsThenConvert(
I.getType(), TrueShadow,
2676 if (ShouldTrackOrigins) {
2677 Shadows.push_back(TrueShadow);
2678 Shadows.push_back(FalseShadow);
2679 Origins.push_back(TrueOrigin);
2680 Origins.push_back(FalseOrigin);
2683 if (TrueShadow == FalseShadow) {
2684 ShadowSel = TrueShadow;
2685 if (ShouldTrackOrigins) {
2686 Shadows.push_back(TrueShadow);
2687 Origins.push_back(TrueOrigin);
2691 SelectInst::Create(
I.getCondition(), TrueShadow, FalseShadow,
"", &
I);
2692 if (ShouldTrackOrigins) {
2693 Shadows.push_back(ShadowSel);
2694 Origins.push_back(SelectInst::Create(
I.getCondition(), TrueOrigin,
2695 FalseOrigin,
"", &
I));
2700 ? DFSF.combineShadowsThenConvert(
2701 I.getType(), CondShadow, ShadowSel, &
I)
2703 if (ShouldTrackOrigins) {
2705 Shadows.push_back(CondShadow);
2706 Origins.push_back(DFSF.getOrigin(
I.getCondition()));
2708 DFSF.setOrigin(&
I, DFSF.combineOrigins(Shadows, Origins, &
I));
2712 void DFSanVisitor::visitMemSetInst(
MemSetInst &
I) {
2714 Value *ValShadow = DFSF.getShadow(
I.getValue());
2715 Value *ValOrigin = DFSF.DFS.shouldTrackOrigins()
2716 ? DFSF.getOrigin(
I.getValue())
2717 : DFSF.DFS.ZeroOrigin;
2719 DFSF.DFS.DFSanSetLabelFn,
2720 {ValShadow, ValOrigin,
2721 IRB.CreateBitCast(I.getDest(), Type::getInt8PtrTy(*DFSF.DFS.Ctx)),
2722 IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
2730 if (DFSF.DFS.shouldTrackOrigins()) {
2732 DFSF.DFS.DFSanMemOriginTransferFn,
2733 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2734 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2735 IRB.CreateIntCast(I.getArgOperand(2), DFSF.DFS.IntptrTy, false)});
2738 Value *RawDestShadow = DFSF.DFS.getShadowAddress(
I.getDest(), &
I);
2739 Value *SrcShadow = DFSF.DFS.getShadowAddress(
I.getSource(), &
I);
2742 DFSF.DFS.ShadowWidthBytes));
2743 Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx);
2746 auto *MTI = cast<MemTransferInst>(
2747 IRB.
CreateCall(
I.getFunctionType(),
I.getCalledOperand(),
2748 {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
2749 MTI->setDestAlignment(DFSF.getShadowAlign(
I.getDestAlign().valueOrOne()));
2750 MTI->setSourceAlignment(DFSF.getShadowAlign(
I.getSourceAlign().valueOrOne()));
2752 IRB.
CreateCall(DFSF.DFS.DFSanMemTransferCallbackFn,
2754 IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
2759 if (!
BR.isConditional())
2762 DFSF.addConditionalCallbacksIfEnabled(
BR,
BR.getCondition());
2765 void DFSanVisitor::visitSwitchInst(
SwitchInst &SW) {
2766 DFSF.addConditionalCallbacksIfEnabled(SW, SW.
getCondition());
2771 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
2772 RetVal =
I->getOperand(0);
2774 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
2775 return I->isMustTailCall();
2780 void DFSanVisitor::visitReturnInst(
ReturnInst &RI) {
2788 Type *RT = DFSF.F->getFunctionType()->getReturnType();
2789 unsigned Size = getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
2795 if (DFSF.DFS.shouldTrackOrigins()) {
2803 std::vector<Value *> &
Args,
2811 Args.push_back(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*
I), &CB));
2817 auto *LabelVAAlloca =
2818 new AllocaInst(LabelVATy, getDataLayout().getAllocaAddrSpace(),
2819 "labelva", &DFSF.F->getEntryBlock().front());
2821 for (
unsigned N = 0;
I != CB.
arg_end(); ++
I, ++
N) {
2823 IRB.
CreateStore(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*
I), &CB),
2832 if (!DFSF.LabelReturnAlloca) {
2834 DFSF.DFS.PrimitiveShadowTy, getDataLayout().getAllocaAddrSpace(),
2835 "labelreturn", &DFSF.F->getEntryBlock().front());
2837 Args.push_back(DFSF.LabelReturnAlloca);
2842 std::vector<Value *> &
Args,
2850 Args.push_back(DFSF.getOrigin(*
I));
2856 auto *OriginVAAlloca =
2857 new AllocaInst(OriginVATy, getDataLayout().getAllocaAddrSpace(),
2858 "originva", &DFSF.F->getEntryBlock().front());
2860 for (
unsigned N = 0;
I != CB.
arg_end(); ++
I, ++
N) {
2870 if (!DFSF.OriginReturnAlloca) {
2872 DFSF.DFS.OriginTy, getDataLayout().getAllocaAddrSpace(),
2873 "originreturn", &DFSF.F->getEntryBlock().front());
2875 Args.push_back(DFSF.OriginReturnAlloca);
2881 switch (DFSF.DFS.getWrapperKind(&
F)) {
2882 case DataFlowSanitizer::WK_Warning:
2884 IRB.
CreateCall(DFSF.DFS.DFSanUnimplementedFn,
2886 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &
F);
2887 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
2888 DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
2890 case DataFlowSanitizer::WK_Discard:
2892 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &
F);
2893 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
2894 DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
2896 case DataFlowSanitizer::WK_Functional:
2898 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &
F);
2899 visitInstOperands(CB);
2901 case DataFlowSanitizer::WK_Custom:
2905 CallInst *CI = dyn_cast<CallInst>(&CB);
2909 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2911 TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
2912 std::string CustomFName = ShouldTrackOrigins ?
"__dfso_" :
"__dfsw_";
2913 CustomFName +=
F.getName();
2915 CustomFName, CustomFn.TransformedType);
2917 CustomFn->copyAttributesFrom(&
F);
2921 CustomFn->removeFnAttrs(DFSF.DFS.ReadOnlyNoneAttrs);
2925 std::vector<Value *>
Args;
2934 const unsigned ShadowArgStart =
Args.size();
2935 addShadowArguments(
F, CB,
Args, IRB);
2938 const unsigned OriginArgStart =
Args.size();
2939 if (ShouldTrackOrigins)
2940 addOriginArguments(
F, CB,
Args, IRB);
2954 const unsigned ArgNo = ShadowArgStart +
N;
2956 DFSF.DFS.PrimitiveShadowTy)
2958 if (ShouldTrackOrigins) {
2959 const unsigned OriginArgNo = OriginArgStart +
N;
2969 IRB.
CreateLoad(DFSF.DFS.PrimitiveShadowTy, DFSF.LabelReturnAlloca);
2970 DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow(
2972 if (ShouldTrackOrigins) {
2974 IRB.
CreateLoad(DFSF.DFS.OriginTy, DFSF.OriginReturnAlloca);
2975 DFSF.setOrigin(CustomCI, OriginLoad);
2986 void DFSanVisitor::visitCallBase(
CallBase &CB) {
2989 visitInstOperands(CB);
2995 if (
F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
3000 if (UnwrappedFnIt != DFSF.DFS.UnwrappedFnMap.end())
3001 if (visitWrappedCallBase(*UnwrappedFnIt->second, CB))
3006 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3011 unsigned ArgOffset = 0;
3013 if (ShouldTrackOrigins) {
3016 if (
I < DFSF.DFS.NumOfElementsInArgOrgTLS &&
3017 !DFSF.DFS.isZeroShadow(ArgShadow))
3019 DFSF.getArgOriginTLS(
I, IRB));
3036 if (
InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
3037 if (II->getNormalDest()->getSinglePredecessor()) {
3038 Next = &II->getNormalDest()->front();
3041 SplitEdge(II->getParent(), II->getNormalDest(), &DFSF.DT);
3042 Next = &NewBB->
front();
3055 unsigned Size =
DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
3058 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3060 LoadInst *LI = NextIRB.CreateAlignedLoad(
3061 DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.
getType(), NextIRB),
3063 DFSF.SkipInsts.insert(LI);
3064 DFSF.setShadow(&CB, LI);
3065 DFSF.NonZeroChecks.push_back(LI);
3068 if (ShouldTrackOrigins) {
3069 LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.OriginTy,
3070 DFSF.getRetvalOriginTLS(),
"_dfsret_o");
3071 DFSF.SkipInsts.insert(LI);
3072 DFSF.setOrigin(&CB, LI);
3077 void DFSanVisitor::visitPHINode(
PHINode &PN) {
3078 Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
3087 DFSF.setShadow(&PN, ShadowPN);
3090 if (DFSF.DFS.shouldTrackOrigins()) {
3096 DFSF.setOrigin(&PN, OriginPN);
3099 DFSF.PHIFixups.push_back({&PN, ShadowPN, OriginPN});
3103 class DataFlowSanitizerLegacyPass :
public ModulePass {
3105 std::vector<std::string> ABIListFiles;
3110 DataFlowSanitizerLegacyPass(
3111 const std::vector<std::string> &ABIListFiles = std::vector<std::string>())
3115 return DataFlowSanitizer(ABIListFiles).runImpl(
M);
3123 "DataFlowSanitizer: dynamic data flow analysis.",
false,
false)
3127 return new DataFlowSanitizerLegacyPass(ABIListFiles);
3132 if (DataFlowSanitizer(ABIListFiles).
runImpl(
M)) {