LLVM  9.0.0svn
X86SpeculativeLoadHardening.cpp
Go to the documentation of this file.
1 //====- X86SpeculativeLoadHardening.cpp - A Spectre v1 mitigation ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// Provide a pass which mitigates speculative execution attacks which operate
11 /// by speculating incorrectly past some predicate (a type check, bounds check,
12 /// or other condition) to reach a load with invalid inputs and leak the data
13 /// accessed by that load using a side channel out of the speculative domain.
14 ///
15 /// For details on the attacks, see the first variant in both the Project Zero
16 /// writeup and the Spectre paper:
17 /// https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
18 /// https://spectreattack.com/spectre.pdf
19 ///
20 //===----------------------------------------------------------------------===//
21 
22 #include "X86.h"
23 #include "X86InstrBuilder.h"
24 #include "X86InstrInfo.h"
25 #include "X86Subtarget.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/DenseMap.h"
28 #include "llvm/ADT/Optional.h"
29 #include "llvm/ADT/STLExtras.h"
30 #include "llvm/ADT/ScopeExit.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallSet.h"
33 #include "llvm/ADT/SmallVector.h"
35 #include "llvm/ADT/Statistic.h"
50 #include "llvm/IR/DebugLoc.h"
51 #include "llvm/MC/MCSchedule.h"
52 #include "llvm/Pass.h"
54 #include "llvm/Support/Debug.h"
56 #include <algorithm>
57 #include <cassert>
58 #include <iterator>
59 #include <utility>
60 
61 using namespace llvm;
62 
63 #define PASS_KEY "x86-slh"
64 #define DEBUG_TYPE PASS_KEY
65 
66 STATISTIC(NumCondBranchesTraced, "Number of conditional branches traced");
67 STATISTIC(NumBranchesUntraced, "Number of branches unable to trace");
68 STATISTIC(NumAddrRegsHardened,
69  "Number of address mode used registers hardaned");
70 STATISTIC(NumPostLoadRegsHardened,
71  "Number of post-load register values hardened");
72 STATISTIC(NumCallsOrJumpsHardened,
73  "Number of calls or jumps requiring extra hardening");
74 STATISTIC(NumInstsInserted, "Number of instructions inserted");
75 STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
76 
78  "x86-speculative-load-hardening",
79  cl::desc("Force enable speculative load hardening"), cl::init(false),
80  cl::Hidden);
81 
83  PASS_KEY "-lfence",
84  cl::desc(
85  "Use LFENCE along each conditional edge to harden against speculative "
86  "loads rather than conditional movs and poisoned pointers."),
87  cl::init(false), cl::Hidden);
88 
90  PASS_KEY "-post-load",
91  cl::desc("Harden the value loaded *after* it is loaded by "
92  "flushing the loaded bits to 1. This is hard to do "
93  "in general but can be done easily for GPRs."),
94  cl::init(true), cl::Hidden);
95 
97  PASS_KEY "-fence-call-and-ret",
98  cl::desc("Use a full speculation fence to harden both call and ret edges "
99  "rather than a lighter weight mitigation."),
100  cl::init(false), cl::Hidden);
101 
103  PASS_KEY "-ip",
104  cl::desc("Harden interprocedurally by passing our state in and out of "
105  "functions in the high bits of the stack pointer."),
106  cl::init(true), cl::Hidden);
107 
108 static cl::opt<bool>
109  HardenLoads(PASS_KEY "-loads",
110  cl::desc("Sanitize loads from memory. When disable, no "
111  "significant security is provided."),
112  cl::init(true), cl::Hidden);
113 
115  PASS_KEY "-indirect",
116  cl::desc("Harden indirect calls and jumps against using speculatively "
117  "stored attacker controlled addresses. This is designed to "
118  "mitigate Spectre v1.2 style attacks."),
119  cl::init(true), cl::Hidden);
120 
121 namespace {
122 
123 class X86SpeculativeLoadHardeningPass : public MachineFunctionPass {
124 public:
125  X86SpeculativeLoadHardeningPass() : MachineFunctionPass(ID) {
128  }
129 
130  StringRef getPassName() const override {
131  return "X86 speculative load hardening";
132  }
133  bool runOnMachineFunction(MachineFunction &MF) override;
134  void getAnalysisUsage(AnalysisUsage &AU) const override;
135 
136  /// Pass identification, replacement for typeid.
137  static char ID;
138 
139 private:
140  /// The information about a block's conditional terminators needed to trace
141  /// our predicate state through the exiting edges.
142  struct BlockCondInfo {
143  MachineBasicBlock *MBB;
144 
145  // We mostly have one conditional branch, and in extremely rare cases have
146  // two. Three and more are so rare as to be unimportant for compile time.
148 
149  MachineInstr *UncondBr;
150  };
151 
152  /// Manages the predicate state traced through the program.
153  struct PredState {
154  unsigned InitialReg;
155  unsigned PoisonReg;
156 
157  const TargetRegisterClass *RC;
159 
160  PredState(MachineFunction &MF, const TargetRegisterClass *RC)
161  : RC(RC), SSA(MF) {}
162  };
163 
164  const X86Subtarget *Subtarget;
166  const X86InstrInfo *TII;
167  const TargetRegisterInfo *TRI;
168 
170 
171  void hardenEdgesWithLFENCE(MachineFunction &MF);
172 
173  SmallVector<BlockCondInfo, 16> collectBlockCondInfo(MachineFunction &MF);
174 
176  tracePredStateThroughCFG(MachineFunction &MF, ArrayRef<BlockCondInfo> Infos);
177 
178  void unfoldCallAndJumpLoads(MachineFunction &MF);
179 
181  tracePredStateThroughIndirectBranches(MachineFunction &MF);
182 
183  void tracePredStateThroughBlocksAndHarden(MachineFunction &MF);
184 
185  unsigned saveEFLAGS(MachineBasicBlock &MBB,
186  MachineBasicBlock::iterator InsertPt, DebugLoc Loc);
187  void restoreEFLAGS(MachineBasicBlock &MBB,
189  unsigned OFReg);
190 
191  void mergePredStateIntoSP(MachineBasicBlock &MBB,
193  unsigned PredStateReg);
194  unsigned extractPredStateFromSP(MachineBasicBlock &MBB,
196  DebugLoc Loc);
197 
198  void
199  hardenLoadAddr(MachineInstr &MI, MachineOperand &BaseMO,
200  MachineOperand &IndexMO,
201  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
202  MachineInstr *
203  sinkPostLoadHardenedInst(MachineInstr &MI,
204  SmallPtrSetImpl<MachineInstr *> &HardenedInstrs);
205  bool canHardenRegister(unsigned Reg);
206  unsigned hardenValueInRegister(unsigned Reg, MachineBasicBlock &MBB,
208  DebugLoc Loc);
209  unsigned hardenPostLoad(MachineInstr &MI);
210  void hardenReturnInstr(MachineInstr &MI);
211  void tracePredStateThroughCall(MachineInstr &MI);
212  void hardenIndirectCallOrJumpInstr(
213  MachineInstr &MI,
214  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
215 };
216 
217 } // end anonymous namespace
218 
220 
221 void X86SpeculativeLoadHardeningPass::getAnalysisUsage(
222  AnalysisUsage &AU) const {
224 }
225 
227  MachineBasicBlock &Succ, int SuccCount,
228  MachineInstr *Br, MachineInstr *&UncondBr,
229  const X86InstrInfo &TII) {
230  assert(!Succ.isEHPad() && "Shouldn't get edges to EH pads!");
231 
232  MachineFunction &MF = *MBB.getParent();
233 
234  MachineBasicBlock &NewMBB = *MF.CreateMachineBasicBlock();
235 
236  // We have to insert the new block immediately after the current one as we
237  // don't know what layout-successor relationships the successor has and we
238  // may not be able to (and generally don't want to) try to fix those up.
239  MF.insert(std::next(MachineFunction::iterator(&MBB)), &NewMBB);
240 
241  // Update the branch instruction if necessary.
242  if (Br) {
243  assert(Br->getOperand(0).getMBB() == &Succ &&
244  "Didn't start with the right target!");
245  Br->getOperand(0).setMBB(&NewMBB);
246 
247  // If this successor was reached through a branch rather than fallthrough,
248  // we might have *broken* fallthrough and so need to inject a new
249  // unconditional branch.
250  if (!UncondBr) {
251  MachineBasicBlock &OldLayoutSucc =
252  *std::next(MachineFunction::iterator(&NewMBB));
253  assert(MBB.isSuccessor(&OldLayoutSucc) &&
254  "Without an unconditional branch, the old layout successor should "
255  "be an actual successor!");
256  auto BrBuilder =
257  BuildMI(&MBB, DebugLoc(), TII.get(X86::JMP_1)).addMBB(&OldLayoutSucc);
258  // Update the unconditional branch now that we've added one.
259  UncondBr = &*BrBuilder;
260  }
261 
262  // Insert unconditional "jump Succ" instruction in the new block if
263  // necessary.
264  if (!NewMBB.isLayoutSuccessor(&Succ)) {
266  TII.insertBranch(NewMBB, &Succ, nullptr, Cond, Br->getDebugLoc());
267  }
268  } else {
269  assert(!UncondBr &&
270  "Cannot have a branchless successor and an unconditional branch!");
271  assert(NewMBB.isLayoutSuccessor(&Succ) &&
272  "A non-branch successor must have been a layout successor before "
273  "and now is a layout successor of the new block.");
274  }
275 
276  // If this is the only edge to the successor, we can just replace it in the
277  // CFG. Otherwise we need to add a new entry in the CFG for the new
278  // successor.
279  if (SuccCount == 1) {
280  MBB.replaceSuccessor(&Succ, &NewMBB);
281  } else {
282  MBB.splitSuccessor(&Succ, &NewMBB);
283  }
284 
285  // Hook up the edge from the new basic block to the old successor in the CFG.
286  NewMBB.addSuccessor(&Succ);
287 
288  // Fix PHI nodes in Succ so they refer to NewMBB instead of MBB.
289  for (MachineInstr &MI : Succ) {
290  if (!MI.isPHI())
291  break;
292  for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps;
293  OpIdx += 2) {
294  MachineOperand &OpV = MI.getOperand(OpIdx);
295  MachineOperand &OpMBB = MI.getOperand(OpIdx + 1);
296  assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!");
297  if (OpMBB.getMBB() != &MBB)
298  continue;
299 
300  // If this is the last edge to the succesor, just replace MBB in the PHI
301  if (SuccCount == 1) {
302  OpMBB.setMBB(&NewMBB);
303  break;
304  }
305 
306  // Otherwise, append a new pair of operands for the new incoming edge.
307  MI.addOperand(MF, OpV);
308  MI.addOperand(MF, MachineOperand::CreateMBB(&NewMBB));
309  break;
310  }
311  }
312 
313  // Inherit live-ins from the successor
314  for (auto &LI : Succ.liveins())
315  NewMBB.addLiveIn(LI);
316 
317  LLVM_DEBUG(dbgs() << " Split edge from '" << MBB.getName() << "' to '"
318  << Succ.getName() << "'.\n");
319  return NewMBB;
320 }
321 
322 /// Removing duplicate PHI operands to leave the PHI in a canonical and
323 /// predictable form.
324 ///
325 /// FIXME: It's really frustrating that we have to do this, but SSA-form in MIR
326 /// isn't what you might expect. We may have multiple entries in PHI nodes for
327 /// a single predecessor. This makes CFG-updating extremely complex, so here we
328 /// simplify all PHI nodes to a model even simpler than the IR's model: exactly
329 /// one entry per predecessor, regardless of how many edges there are.
332  SmallVector<int, 4> DupIndices;
333  for (auto &MBB : MF)
334  for (auto &MI : MBB) {
335  if (!MI.isPHI())
336  break;
337 
338  // First we scan the operands of the PHI looking for duplicate entries
339  // a particular predecessor. We retain the operand index of each duplicate
340  // entry found.
341  for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps;
342  OpIdx += 2)
343  if (!Preds.insert(MI.getOperand(OpIdx + 1).getMBB()).second)
344  DupIndices.push_back(OpIdx);
345 
346  // Now walk the duplicate indices, removing both the block and value. Note
347  // that these are stored as a vector making this element-wise removal
348  // :w
349  // potentially quadratic.
350  //
351  // FIXME: It is really frustrating that we have to use a quadratic
352  // removal algorithm here. There should be a better way, but the use-def
353  // updates required make that impossible using the public API.
354  //
355  // Note that we have to process these backwards so that we don't
356  // invalidate other indices with each removal.
357  while (!DupIndices.empty()) {
358  int OpIdx = DupIndices.pop_back_val();
359  // Remove both the block and value operand, again in reverse order to
360  // preserve indices.
361  MI.RemoveOperand(OpIdx + 1);
362  MI.RemoveOperand(OpIdx);
363  }
364 
365  Preds.clear();
366  }
367 }
368 
369 /// Helper to scan a function for loads vulnerable to misspeculation that we
370 /// want to harden.
371 ///
372 /// We use this to avoid making changes to functions where there is nothing we
373 /// need to do to harden against misspeculation.
375  for (MachineBasicBlock &MBB : MF) {
376  for (MachineInstr &MI : MBB) {
377  // Loads within this basic block after an LFENCE are not at risk of
378  // speculatively executing with invalid predicates from prior control
379  // flow. So break out of this block but continue scanning the function.
380  if (MI.getOpcode() == X86::LFENCE)
381  break;
382 
383  // Looking for loads only.
384  if (!MI.mayLoad())
385  continue;
386 
387  // An MFENCE is modeled as a load but isn't vulnerable to misspeculation.
388  if (MI.getOpcode() == X86::MFENCE)
389  continue;
390 
391  // We found a load.
392  return true;
393  }
394  }
395 
396  // No loads found.
397  return false;
398 }
399 
400 bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
401  MachineFunction &MF) {
402  LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
403  << " **********\n");
404 
405  // Only run if this pass is forced enabled or we detect the relevant function
406  // attribute requesting SLH.
408  !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
409  return false;
410 
411  Subtarget = &MF.getSubtarget<X86Subtarget>();
412  MRI = &MF.getRegInfo();
413  TII = Subtarget->getInstrInfo();
414  TRI = Subtarget->getRegisterInfo();
415 
416  // FIXME: Support for 32-bit.
417  PS.emplace(MF, &X86::GR64_NOSPRegClass);
418 
419  if (MF.begin() == MF.end())
420  // Nothing to do for a degenerate empty function...
421  return false;
422 
423  // We support an alternative hardening technique based on a debug flag.
424  if (HardenEdgesWithLFENCE) {
425  hardenEdgesWithLFENCE(MF);
426  return true;
427  }
428 
429  // Create a dummy debug loc to use for all the generated code here.
430  DebugLoc Loc;
431 
432  MachineBasicBlock &Entry = *MF.begin();
433  auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());
434 
435  // Do a quick scan to see if we have any checkable loads.
436  bool HasVulnerableLoad = hasVulnerableLoad(MF);
437 
438  // See if we have any conditional branching blocks that we will need to trace
439  // predicate state through.
440  SmallVector<BlockCondInfo, 16> Infos = collectBlockCondInfo(MF);
441 
442  // If we have no interesting conditions or loads, nothing to do here.
443  if (!HasVulnerableLoad && Infos.empty())
444  return true;
445 
446  // The poison value is required to be an all-ones value for many aspects of
447  // this mitigation.
448  const int PoisonVal = -1;
449  PS->PoisonReg = MRI->createVirtualRegister(PS->RC);
450  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64ri32), PS->PoisonReg)
451  .addImm(PoisonVal);
452  ++NumInstsInserted;
453 
454  // If we have loads being hardened and we've asked for call and ret edges to
455  // get a full fence-based mitigation, inject that fence.
456  if (HasVulnerableLoad && FenceCallAndRet) {
457  // We need to insert an LFENCE at the start of the function to suspend any
458  // incoming misspeculation from the caller. This helps two-fold: the caller
459  // may not have been protected as this code has been, and this code gets to
460  // not take any specific action to protect across calls.
461  // FIXME: We could skip this for functions which unconditionally return
462  // a constant.
463  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::LFENCE));
464  ++NumInstsInserted;
465  ++NumLFENCEsInserted;
466  }
467 
468  // If we guarded the entry with an LFENCE and have no conditionals to protect
469  // in blocks, then we're done.
470  if (FenceCallAndRet && Infos.empty())
471  // We may have changed the function's code at this point to insert fences.
472  return true;
473 
474  // For every basic block in the function which can b
476  // Set up the predicate state by extracting it from the incoming stack
477  // pointer so we pick up any misspeculation in our caller.
478  PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
479  } else {
480  // Otherwise, just build the predicate state itself by zeroing a register
481  // as we don't need any initial state.
482  PS->InitialReg = MRI->createVirtualRegister(PS->RC);
483  unsigned PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass);
484  auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0),
485  PredStateSubReg);
486  ++NumInstsInserted;
487  MachineOperand *ZeroEFLAGSDefOp =
488  ZeroI->findRegisterDefOperand(X86::EFLAGS);
489  assert(ZeroEFLAGSDefOp && ZeroEFLAGSDefOp->isImplicit() &&
490  "Must have an implicit def of EFLAGS!");
491  ZeroEFLAGSDefOp->setIsDead(true);
492  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG),
493  PS->InitialReg)
494  .addImm(0)
495  .addReg(PredStateSubReg)
496  .addImm(X86::sub_32bit);
497  }
498 
499  // We're going to need to trace predicate state throughout the function's
500  // CFG. Prepare for this by setting up our initial state of PHIs with unique
501  // predecessor entries and all the initial predicate state.
503 
504  // Track the updated values in an SSA updater to rewrite into SSA form at the
505  // end.
506  PS->SSA.Initialize(PS->InitialReg);
507  PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);
508 
509  // Trace through the CFG.
510  auto CMovs = tracePredStateThroughCFG(MF, Infos);
511 
512  // We may also enter basic blocks in this function via exception handling
513  // control flow. Here, if we are hardening interprocedurally, we need to
514  // re-capture the predicate state from the throwing code. In the Itanium ABI,
515  // the throw will always look like a call to __cxa_throw and will have the
516  // predicate state in the stack pointer, so extract fresh predicate state from
517  // the stack pointer and make it available in SSA.
518  // FIXME: Handle non-itanium ABI EH models.
520  for (MachineBasicBlock &MBB : MF) {
521  assert(!MBB.isEHScopeEntry() && "Only Itanium ABI EH supported!");
522  assert(!MBB.isEHFuncletEntry() && "Only Itanium ABI EH supported!");
523  assert(!MBB.isCleanupFuncletEntry() && "Only Itanium ABI EH supported!");
524  if (!MBB.isEHPad())
525  continue;
526  PS->SSA.AddAvailableValue(
527  &MBB,
528  extractPredStateFromSP(MBB, MBB.SkipPHIsAndLabels(MBB.begin()), Loc));
529  }
530  }
531 
533  // If we are going to harden calls and jumps we need to unfold their memory
534  // operands.
535  unfoldCallAndJumpLoads(MF);
536 
537  // Then we trace predicate state through the indirect branches.
538  auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);
539  CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());
540  }
541 
542  // Now that we have the predicate state available at the start of each block
543  // in the CFG, trace it through each block, hardening vulnerable instructions
544  // as we go.
545  tracePredStateThroughBlocksAndHarden(MF);
546 
547  // Now rewrite all the uses of the pred state using the SSA updater to insert
548  // PHIs connecting the state between blocks along the CFG edges.
549  for (MachineInstr *CMovI : CMovs)
550  for (MachineOperand &Op : CMovI->operands()) {
551  if (!Op.isReg() || Op.getReg() != PS->InitialReg)
552  continue;
553 
554  PS->SSA.RewriteUse(Op);
555  }
556 
557  LLVM_DEBUG(dbgs() << "Final speculative load hardened function:\n"; MF.dump();
558  dbgs() << "\n"; MF.verify(this));
559  return true;
560 }
561 
562 /// Implements the naive hardening approach of putting an LFENCE after every
563 /// potentially mis-predicted control flow construct.
564 ///
565 /// We include this as an alternative mostly for the purpose of comparison. The
566 /// performance impact of this is expected to be extremely severe and not
567 /// practical for any real-world users.
568 void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(
569  MachineFunction &MF) {
570  // First, we scan the function looking for blocks that are reached along edges
571  // that we might want to harden.
573  for (MachineBasicBlock &MBB : MF) {
574  // If there are no or only one successor, nothing to do here.
575  if (MBB.succ_size() <= 1)
576  continue;
577 
578  // Skip blocks unless their terminators start with a branch. Other
579  // terminators don't seem interesting for guarding against misspeculation.
580  auto TermIt = MBB.getFirstTerminator();
581  if (TermIt == MBB.end() || !TermIt->isBranch())
582  continue;
583 
584  // Add all the non-EH-pad succossors to the blocks we want to harden. We
585  // skip EH pads because there isn't really a condition of interest on
586  // entering.
587  for (MachineBasicBlock *SuccMBB : MBB.successors())
588  if (!SuccMBB->isEHPad())
589  Blocks.insert(SuccMBB);
590  }
591 
592  for (MachineBasicBlock *MBB : Blocks) {
593  auto InsertPt = MBB->SkipPHIsAndLabels(MBB->begin());
594  BuildMI(*MBB, InsertPt, DebugLoc(), TII->get(X86::LFENCE));
595  ++NumInstsInserted;
596  ++NumLFENCEsInserted;
597  }
598 }
599 
601 X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) {
603 
604  // Walk the function and build up a summary for each block's conditions that
605  // we need to trace through.
606  for (MachineBasicBlock &MBB : MF) {
607  // If there are no or only one successor, nothing to do here.
608  if (MBB.succ_size() <= 1)
609  continue;
610 
611  // We want to reliably handle any conditional branch terminators in the
612  // MBB, so we manually analyze the branch. We can handle all of the
613  // permutations here, including ones that analyze branch cannot.
614  //
615  // The approach is to walk backwards across the terminators, resetting at
616  // any unconditional non-indirect branch, and track all conditional edges
617  // to basic blocks as well as the fallthrough or unconditional successor
618  // edge. For each conditional edge, we track the target and the opposite
619  // condition code in order to inject a "no-op" cmov into that successor
620  // that will harden the predicate. For the fallthrough/unconditional
621  // edge, we inject a separate cmov for each conditional branch with
622  // matching condition codes. This effectively implements an "and" of the
623  // condition flags, even if there isn't a single condition flag that would
624  // directly implement that. We don't bother trying to optimize either of
625  // these cases because if such an optimization is possible, LLVM should
626  // have optimized the conditional *branches* in that way already to reduce
627  // instruction count. This late, we simply assume the minimal number of
628  // branch instructions is being emitted and use that to guide our cmov
629  // insertion.
630 
631  BlockCondInfo Info = {&MBB, {}, nullptr};
632 
633  // Now walk backwards through the terminators and build up successors they
634  // reach and the conditions.
635  for (MachineInstr &MI : llvm::reverse(MBB)) {
636  // Once we've handled all the terminators, we're done.
637  if (!MI.isTerminator())
638  break;
639 
640  // If we see a non-branch terminator, we can't handle anything so bail.
641  if (!MI.isBranch()) {
642  Info.CondBrs.clear();
643  break;
644  }
645 
646  // If we see an unconditional branch, reset our state, clear any
647  // fallthrough, and set this is the "else" successor.
648  if (MI.getOpcode() == X86::JMP_1) {
649  Info.CondBrs.clear();
650  Info.UncondBr = &MI;
651  continue;
652  }
653 
654  // If we get an invalid condition, we have an indirect branch or some
655  // other unanalyzable "fallthrough" case. We model this as a nullptr for
656  // the destination so we can still guard any conditional successors.
657  // Consider code sequences like:
658  // ```
659  // jCC L1
660  // jmpq *%rax
661  // ```
662  // We still want to harden the edge to `L1`.
664  Info.CondBrs.clear();
665  Info.UncondBr = &MI;
666  continue;
667  }
668 
669  // We have a vanilla conditional branch, add it to our list.
670  Info.CondBrs.push_back(&MI);
671  }
672  if (Info.CondBrs.empty()) {
673  ++NumBranchesUntraced;
674  LLVM_DEBUG(dbgs() << "WARNING: unable to secure successors of block:\n";
675  MBB.dump());
676  continue;
677  }
678 
679  Infos.push_back(Info);
680  }
681 
682  return Infos;
683 }
684 
685 /// Trace the predicate state through the CFG, instrumenting each conditional
686 /// branch such that misspeculation through an edge will poison the predicate
687 /// state.
688 ///
689 /// Returns the list of inserted CMov instructions so that they can have their
690 /// uses of the predicate state rewritten into proper SSA form once it is
691 /// complete.
693 X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(
695  // Collect the inserted cmov instructions so we can rewrite their uses of the
696  // predicate state into SSA form.
698 
699  // Now walk all of the basic blocks looking for ones that end in conditional
700  // jumps where we need to update this register along each edge.
701  for (const BlockCondInfo &Info : Infos) {
702  MachineBasicBlock &MBB = *Info.MBB;
703  const SmallVectorImpl<MachineInstr *> &CondBrs = Info.CondBrs;
704  MachineInstr *UncondBr = Info.UncondBr;
705 
706  LLVM_DEBUG(dbgs() << "Tracing predicate through block: " << MBB.getName()
707  << "\n");
708  ++NumCondBranchesTraced;
709 
710  // Compute the non-conditional successor as either the target of any
711  // unconditional branch or the layout successor.
712  MachineBasicBlock *UncondSucc =
713  UncondBr ? (UncondBr->getOpcode() == X86::JMP_1
714  ? UncondBr->getOperand(0).getMBB()
715  : nullptr)
716  : &*std::next(MachineFunction::iterator(&MBB));
717 
718  // Count how many edges there are to any given successor.
720  if (UncondSucc)
721  ++SuccCounts[UncondSucc];
722  for (auto *CondBr : CondBrs)
723  ++SuccCounts[CondBr->getOperand(0).getMBB()];
724 
725  // A lambda to insert cmov instructions into a block checking all of the
726  // condition codes in a sequence.
727  auto BuildCheckingBlockForSuccAndConds =
728  [&](MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount,
729  MachineInstr *Br, MachineInstr *&UncondBr,
730  ArrayRef<X86::CondCode> Conds) {
731  // First, we split the edge to insert the checking block into a safe
732  // location.
733  auto &CheckingMBB =
734  (SuccCount == 1 && Succ.pred_size() == 1)
735  ? Succ
736  : splitEdge(MBB, Succ, SuccCount, Br, UncondBr, *TII);
737 
738  bool LiveEFLAGS = Succ.isLiveIn(X86::EFLAGS);
739  if (!LiveEFLAGS)
740  CheckingMBB.addLiveIn(X86::EFLAGS);
741 
742  // Now insert the cmovs to implement the checks.
743  auto InsertPt = CheckingMBB.begin();
744  assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&
745  "Should never have a PHI in the initial checking block as it "
746  "always has a single predecessor!");
747 
748  // We will wire each cmov to each other, but need to start with the
749  // incoming pred state.
750  unsigned CurStateReg = PS->InitialReg;
751 
752  for (X86::CondCode Cond : Conds) {
753  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
754  auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
755 
756  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
757  // Note that we intentionally use an empty debug location so that
758  // this picks up the preceding location.
759  auto CMovI = BuildMI(CheckingMBB, InsertPt, DebugLoc(),
760  TII->get(CMovOp), UpdatedStateReg)
761  .addReg(CurStateReg)
762  .addReg(PS->PoisonReg)
763  .addImm(Cond);
764  // If this is the last cmov and the EFLAGS weren't originally
765  // live-in, mark them as killed.
766  if (!LiveEFLAGS && Cond == Conds.back())
767  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
768 
769  ++NumInstsInserted;
770  LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump();
771  dbgs() << "\n");
772 
773  // The first one of the cmovs will be using the top level
774  // `PredStateReg` and need to get rewritten into SSA form.
775  if (CurStateReg == PS->InitialReg)
776  CMovs.push_back(&*CMovI);
777 
778  // The next cmov should start from this one's def.
779  CurStateReg = UpdatedStateReg;
780  }
781 
782  // And put the last one into the available values for SSA form of our
783  // predicate state.
784  PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);
785  };
786 
787  std::vector<X86::CondCode> UncondCodeSeq;
788  for (auto *CondBr : CondBrs) {
789  MachineBasicBlock &Succ = *CondBr->getOperand(0).getMBB();
790  int &SuccCount = SuccCounts[&Succ];
791 
792  X86::CondCode Cond = X86::getCondFromBranch(*CondBr);
794  UncondCodeSeq.push_back(Cond);
795 
796  BuildCheckingBlockForSuccAndConds(MBB, Succ, SuccCount, CondBr, UncondBr,
797  {InvCond});
798 
799  // Decrement the successor count now that we've split one of the edges.
800  // We need to keep the count of edges to the successor accurate in order
801  // to know above when to *replace* the successor in the CFG vs. just
802  // adding the new successor.
803  --SuccCount;
804  }
805 
806  // Since we may have split edges and changed the number of successors,
807  // normalize the probabilities. This avoids doing it each time we split an
808  // edge.
809  MBB.normalizeSuccProbs();
810 
811  // Finally, we need to insert cmovs into the "fallthrough" edge. Here, we
812  // need to intersect the other condition codes. We can do this by just
813  // doing a cmov for each one.
814  if (!UncondSucc)
815  // If we have no fallthrough to protect (perhaps it is an indirect jump?)
816  // just skip this and continue.
817  continue;
818 
819  assert(SuccCounts[UncondSucc] == 1 &&
820  "We should never have more than one edge to the unconditional "
821  "successor at this point because every other edge must have been "
822  "split above!");
823 
824  // Sort and unique the codes to minimize them.
825  llvm::sort(UncondCodeSeq);
826  UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()),
827  UncondCodeSeq.end());
828 
829  // Build a checking version of the successor.
830  BuildCheckingBlockForSuccAndConds(MBB, *UncondSucc, /*SuccCount*/ 1,
831  UncondBr, UncondBr, UncondCodeSeq);
832  }
833 
834  return CMovs;
835 }
836 
837 /// Compute the register class for the unfolded load.
838 ///
839 /// FIXME: This should probably live in X86InstrInfo, potentially by adding
840 /// a way to unfold into a newly created vreg rather than requiring a register
841 /// input.
842 static const TargetRegisterClass *
844  unsigned Opcode) {
845  unsigned Index;
846  unsigned UnfoldedOpc = TII.getOpcodeAfterMemoryUnfold(
847  Opcode, /*UnfoldLoad*/ true, /*UnfoldStore*/ false, &Index);
848  const MCInstrDesc &MCID = TII.get(UnfoldedOpc);
849  return TII.getRegClass(MCID, Index, &TII.getRegisterInfo(), MF);
850 }
851 
852 void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
853  MachineFunction &MF) {
854  for (MachineBasicBlock &MBB : MF)
855  for (auto MII = MBB.instr_begin(), MIE = MBB.instr_end(); MII != MIE;) {
856  // Grab a reference and increment the iterator so we can remove this
857  // instruction if needed without disturbing the iteration.
858  MachineInstr &MI = *MII++;
859 
860  // Must either be a call or a branch.
861  if (!MI.isCall() && !MI.isBranch())
862  continue;
863  // We only care about loading variants of these instructions.
864  if (!MI.mayLoad())
865  continue;
866 
867  switch (MI.getOpcode()) {
868  default: {
869  LLVM_DEBUG(
870  dbgs() << "ERROR: Found an unexpected loading branch or call "
871  "instruction:\n";
872  MI.dump(); dbgs() << "\n");
873  report_fatal_error("Unexpected loading branch or call!");
874  }
875 
876  case X86::FARCALL16m:
877  case X86::FARCALL32m:
878  case X86::FARCALL64:
879  case X86::FARJMP16m:
880  case X86::FARJMP32m:
881  case X86::FARJMP64:
882  // We cannot mitigate far jumps or calls, but we also don't expect them
883  // to be vulnerable to Spectre v1.2 style attacks.
884  continue;
885 
886  case X86::CALL16m:
887  case X86::CALL16m_NT:
888  case X86::CALL32m:
889  case X86::CALL32m_NT:
890  case X86::CALL64m:
891  case X86::CALL64m_NT:
892  case X86::JMP16m:
893  case X86::JMP16m_NT:
894  case X86::JMP32m:
895  case X86::JMP32m_NT:
896  case X86::JMP64m:
897  case X86::JMP64m_NT:
898  case X86::TAILJMPm64:
899  case X86::TAILJMPm64_REX:
900  case X86::TAILJMPm:
901  case X86::TCRETURNmi64:
902  case X86::TCRETURNmi: {
903  // Use the generic unfold logic now that we know we're dealing with
904  // expected instructions.
905  // FIXME: We don't have test coverage for all of these!
906  auto *UnfoldedRC = getRegClassForUnfoldedLoad(MF, *TII, MI.getOpcode());
907  if (!UnfoldedRC) {
908  LLVM_DEBUG(dbgs()
909  << "ERROR: Unable to unfold load from instruction:\n";
910  MI.dump(); dbgs() << "\n");
911  report_fatal_error("Unable to unfold load!");
912  }
913  unsigned Reg = MRI->createVirtualRegister(UnfoldedRC);
915  // If we were able to compute an unfolded reg class, any failure here
916  // is just a programming error so just assert.
917  bool Unfolded =
918  TII->unfoldMemoryOperand(MF, MI, Reg, /*UnfoldLoad*/ true,
919  /*UnfoldStore*/ false, NewMIs);
920  (void)Unfolded;
921  assert(Unfolded &&
922  "Computed unfolded register class but failed to unfold");
923  // Now stitch the new instructions into place and erase the old one.
924  for (auto *NewMI : NewMIs)
925  MBB.insert(MI.getIterator(), NewMI);
926  MI.eraseFromParent();
927  LLVM_DEBUG({
928  dbgs() << "Unfolded load successfully into:\n";
929  for (auto *NewMI : NewMIs) {
930  NewMI->dump();
931  dbgs() << "\n";
932  }
933  });
934  continue;
935  }
936  }
937  llvm_unreachable("Escaped switch with default!");
938  }
939 }
940 
941 /// Trace the predicate state through indirect branches, instrumenting them to
942 /// poison the state if a target is reached that does not match the expected
943 /// target.
944 ///
945 /// This is designed to mitigate Spectre variant 1 attacks where an indirect
946 /// branch is trained to predict a particular target and then mispredicts that
947 /// target in a way that can leak data. Despite using an indirect branch, this
948 /// is really a variant 1 style attack: it does not steer execution to an
949 /// arbitrary or attacker controlled address, and it does not require any
950 /// special code executing next to the victim. This attack can also be mitigated
951 /// through retpolines, but those require either replacing indirect branches
952 /// with conditional direct branches or lowering them through a device that
953 /// blocks speculation. This mitigation can replace these retpoline-style
954 /// mitigations for jump tables and other indirect branches within a function
955 /// when variant 2 isn't a risk while allowing limited speculation. Indirect
956 /// calls, however, cannot be mitigated through this technique without changing
957 /// the ABI in a fundamental way.
959 X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(
960  MachineFunction &MF) {
961  // We use the SSAUpdater to insert PHI nodes for the target addresses of
962  // indirect branches. We don't actually need the full power of the SSA updater
963  // in this particular case as we always have immediately available values, but
964  // this avoids us having to re-implement the PHI construction logic.
965  MachineSSAUpdater TargetAddrSSA(MF);
966  TargetAddrSSA.Initialize(MRI->createVirtualRegister(&X86::GR64RegClass));
967 
968  // Track which blocks were terminated with an indirect branch.
969  SmallPtrSet<MachineBasicBlock *, 4> IndirectTerminatedMBBs;
970 
971  // We need to know what blocks end up reached via indirect branches. We
972  // expect this to be a subset of those whose address is taken and so track it
973  // directly via the CFG.
974  SmallPtrSet<MachineBasicBlock *, 4> IndirectTargetMBBs;
975 
976  // Walk all the blocks which end in an indirect branch and make the
977  // target address available.
978  for (MachineBasicBlock &MBB : MF) {
979  // Find the last terminator.
980  auto MII = MBB.instr_rbegin();
981  while (MII != MBB.instr_rend() && MII->isDebugInstr())
982  ++MII;
983  if (MII == MBB.instr_rend())
984  continue;
985  MachineInstr &TI = *MII;
986  if (!TI.isTerminator() || !TI.isBranch())
987  // No terminator or non-branch terminator.
988  continue;
989 
990  unsigned TargetReg;
991 
992  switch (TI.getOpcode()) {
993  default:
994  // Direct branch or conditional branch (leading to fallthrough).
995  continue;
996 
997  case X86::FARJMP16m:
998  case X86::FARJMP32m:
999  case X86::FARJMP64:
1000  // We cannot mitigate far jumps or calls, but we also don't expect them
1001  // to be vulnerable to Spectre v1.2 or v2 (self trained) style attacks.
1002  continue;
1003 
1004  case X86::JMP16m:
1005  case X86::JMP16m_NT:
1006  case X86::JMP32m:
1007  case X86::JMP32m_NT:
1008  case X86::JMP64m:
1009  case X86::JMP64m_NT:
1010  // Mostly as documentation.
1011  report_fatal_error("Memory operand jumps should have been unfolded!");
1012 
1013  case X86::JMP16r:
1015  "Support for 16-bit indirect branches is not implemented.");
1016  case X86::JMP32r:
1018  "Support for 32-bit indirect branches is not implemented.");
1019 
1020  case X86::JMP64r:
1021  TargetReg = TI.getOperand(0).getReg();
1022  }
1023 
1024  // We have definitely found an indirect branch. Verify that there are no
1025  // preceding conditional branches as we don't yet support that.
1026  if (llvm::any_of(MBB.terminators(), [&](MachineInstr &OtherTI) {
1027  return !OtherTI.isDebugInstr() && &OtherTI != &TI;
1028  })) {
1029  LLVM_DEBUG({
1030  dbgs() << "ERROR: Found other terminators in a block with an indirect "
1031  "branch! This is not yet supported! Terminator sequence:\n";
1032  for (MachineInstr &MI : MBB.terminators()) {
1033  MI.dump();
1034  dbgs() << '\n';
1035  }
1036  });
1037  report_fatal_error("Unimplemented terminator sequence!");
1038  }
1039 
1040  // Make the target register an available value for this block.
1041  TargetAddrSSA.AddAvailableValue(&MBB, TargetReg);
1042  IndirectTerminatedMBBs.insert(&MBB);
1043 
1044  // Add all the successors to our target candidates.
1045  for (MachineBasicBlock *Succ : MBB.successors())
1046  IndirectTargetMBBs.insert(Succ);
1047  }
1048 
1049  // Keep track of the cmov instructions we insert so we can return them.
1051 
1052  // If we didn't find any indirect branches with targets, nothing to do here.
1053  if (IndirectTargetMBBs.empty())
1054  return CMovs;
1055 
1056  // We found indirect branches and targets that need to be instrumented to
1057  // harden loads within them. Walk the blocks of the function (to get a stable
1058  // ordering) and instrument each target of an indirect branch.
1059  for (MachineBasicBlock &MBB : MF) {
1060  // Skip the blocks that aren't candidate targets.
1061  if (!IndirectTargetMBBs.count(&MBB))
1062  continue;
1063 
1064  // We don't expect EH pads to ever be reached via an indirect branch. If
1065  // this is desired for some reason, we could simply skip them here rather
1066  // than asserting.
1067  assert(!MBB.isEHPad() &&
1068  "Unexpected EH pad as target of an indirect branch!");
1069 
1070  // We should never end up threading EFLAGS into a block to harden
1071  // conditional jumps as there would be an additional successor via the
1072  // indirect branch. As a consequence, all such edges would be split before
1073  // reaching here, and the inserted block will handle the EFLAGS-based
1074  // hardening.
1075  assert(!MBB.isLiveIn(X86::EFLAGS) &&
1076  "Cannot check within a block that already has live-in EFLAGS!");
1077 
1078  // We can't handle having non-indirect edges into this block unless this is
1079  // the only successor and we can synthesize the necessary target address.
1080  for (MachineBasicBlock *Pred : MBB.predecessors()) {
1081  // If we've already handled this by extracting the target directly,
1082  // nothing to do.
1083  if (IndirectTerminatedMBBs.count(Pred))
1084  continue;
1085 
1086  // Otherwise, we have to be the only successor. We generally expect this
1087  // to be true as conditional branches should have had a critical edge
1088  // split already. We don't however need to worry about EH pad successors
1089  // as they'll happily ignore the target and their hardening strategy is
1090  // resilient to all ways in which they could be reached speculatively.
1091  if (!llvm::all_of(Pred->successors(), [&](MachineBasicBlock *Succ) {
1092  return Succ->isEHPad() || Succ == &MBB;
1093  })) {
1094  LLVM_DEBUG({
1095  dbgs() << "ERROR: Found conditional entry to target of indirect "
1096  "branch!\n";
1097  Pred->dump();
1098  MBB.dump();
1099  });
1100  report_fatal_error("Cannot harden a conditional entry to a target of "
1101  "an indirect branch!");
1102  }
1103 
1104  // Now we need to compute the address of this block and install it as a
1105  // synthetic target in the predecessor. We do this at the bottom of the
1106  // predecessor.
1107  auto InsertPt = Pred->getFirstTerminator();
1108  unsigned TargetReg = MRI->createVirtualRegister(&X86::GR64RegClass);
1109  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
1110  !Subtarget->isPositionIndependent()) {
1111  // Directly materialize it into an immediate.
1112  auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(),
1113  TII->get(X86::MOV64ri32), TargetReg)
1114  .addMBB(&MBB);
1115  ++NumInstsInserted;
1116  (void)AddrI;
1117  LLVM_DEBUG(dbgs() << " Inserting mov: "; AddrI->dump();
1118  dbgs() << "\n");
1119  } else {
1120  auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(), TII->get(X86::LEA64r),
1121  TargetReg)
1122  .addReg(/*Base*/ X86::RIP)
1123  .addImm(/*Scale*/ 1)
1124  .addReg(/*Index*/ 0)
1125  .addMBB(&MBB)
1126  .addReg(/*Segment*/ 0);
1127  ++NumInstsInserted;
1128  (void)AddrI;
1129  LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump();
1130  dbgs() << "\n");
1131  }
1132  // And make this available.
1133  TargetAddrSSA.AddAvailableValue(Pred, TargetReg);
1134  }
1135 
1136  // Materialize the needed SSA value of the target. Note that we need the
1137  // middle of the block as this block might at the bottom have an indirect
1138  // branch back to itself. We can do this here because at this point, every
1139  // predecessor of this block has an available value. This is basically just
1140  // automating the construction of a PHI node for this target.
1141  unsigned TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&MBB);
1142 
1143  // Insert a comparison of the incoming target register with this block's
1144  // address. This also requires us to mark the block as having its address
1145  // taken explicitly.
1146  MBB.setHasAddressTaken();
1147  auto InsertPt = MBB.SkipPHIsLabelsAndDebug(MBB.begin());
1148  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
1149  !Subtarget->isPositionIndependent()) {
1150  // Check directly against a relocated immediate when we can.
1151  auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64ri32))
1152  .addReg(TargetReg, RegState::Kill)
1153  .addMBB(&MBB);
1154  ++NumInstsInserted;
1155  (void)CheckI;
1156  LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
1157  } else {
1158  // Otherwise compute the address into a register first.
1159  unsigned AddrReg = MRI->createVirtualRegister(&X86::GR64RegClass);
1160  auto AddrI =
1161  BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::LEA64r), AddrReg)
1162  .addReg(/*Base*/ X86::RIP)
1163  .addImm(/*Scale*/ 1)
1164  .addReg(/*Index*/ 0)
1165  .addMBB(&MBB)
1166  .addReg(/*Segment*/ 0);
1167  ++NumInstsInserted;
1168  (void)AddrI;
1169  LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump(); dbgs() << "\n");
1170  auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64rr))
1171  .addReg(TargetReg, RegState::Kill)
1172  .addReg(AddrReg, RegState::Kill);
1173  ++NumInstsInserted;
1174  (void)CheckI;
1175  LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
1176  }
1177 
1178  // Now cmov over the predicate if the comparison wasn't equal.
1179  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
1180  auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
1181  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
1182  auto CMovI =
1183  BuildMI(MBB, InsertPt, DebugLoc(), TII->get(CMovOp), UpdatedStateReg)
1184  .addReg(PS->InitialReg)
1185  .addReg(PS->PoisonReg)
1186  .addImm(X86::COND_NE);
1187  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
1188  ++NumInstsInserted;
1189  LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
1190  CMovs.push_back(&*CMovI);
1191 
1192  // And put the new value into the available values for SSA form of our
1193  // predicate state.
1194  PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
1195  }
1196 
1197  // Return all the newly inserted cmov instructions of the predicate state.
1198  return CMovs;
1199 }
1200 
1201 /// Returns true if the instruction has no behavior (specified or otherwise)
1202 /// that is based on the value of any of its register operands
1203 ///
1204 /// A classical example of something that is inherently not data invariant is an
1205 /// indirect jump -- the destination is loaded into icache based on the bits set
1206 /// in the jump destination register.
1207 ///
1208 /// FIXME: This should become part of our instruction tables.
1210  switch (MI.getOpcode()) {
1211  default:
1212  // By default, assume that the instruction is not data invariant.
1213  return false;
1214 
1215  // Some target-independent operations that trivially lower to data-invariant
1216  // instructions.
1217  case TargetOpcode::COPY:
1218  case TargetOpcode::INSERT_SUBREG:
1219  case TargetOpcode::SUBREG_TO_REG:
1220  return true;
1221 
1222  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
1223  // However, they set flags and are perhaps the most surprisingly constant
1224  // time operations so we call them out here separately.
1225  case X86::IMUL16rr:
1226  case X86::IMUL16rri8:
1227  case X86::IMUL16rri:
1228  case X86::IMUL32rr:
1229  case X86::IMUL32rri8:
1230  case X86::IMUL32rri:
1231  case X86::IMUL64rr:
1232  case X86::IMUL64rri32:
1233  case X86::IMUL64rri8:
1234 
1235  // Bit scanning and counting instructions that are somewhat surprisingly
1236  // constant time as they scan across bits and do other fairly complex
1237  // operations like popcnt, but are believed to be constant time on x86.
1238  // However, these set flags.
1239  case X86::BSF16rr:
1240  case X86::BSF32rr:
1241  case X86::BSF64rr:
1242  case X86::BSR16rr:
1243  case X86::BSR32rr:
1244  case X86::BSR64rr:
1245  case X86::LZCNT16rr:
1246  case X86::LZCNT32rr:
1247  case X86::LZCNT64rr:
1248  case X86::POPCNT16rr:
1249  case X86::POPCNT32rr:
1250  case X86::POPCNT64rr:
1251  case X86::TZCNT16rr:
1252  case X86::TZCNT32rr:
1253  case X86::TZCNT64rr:
1254 
1255  // Bit manipulation instructions are effectively combinations of basic
1256  // arithmetic ops, and should still execute in constant time. These also
1257  // set flags.
1258  case X86::BLCFILL32rr:
1259  case X86::BLCFILL64rr:
1260  case X86::BLCI32rr:
1261  case X86::BLCI64rr:
1262  case X86::BLCIC32rr:
1263  case X86::BLCIC64rr:
1264  case X86::BLCMSK32rr:
1265  case X86::BLCMSK64rr:
1266  case X86::BLCS32rr:
1267  case X86::BLCS64rr:
1268  case X86::BLSFILL32rr:
1269  case X86::BLSFILL64rr:
1270  case X86::BLSI32rr:
1271  case X86::BLSI64rr:
1272  case X86::BLSIC32rr:
1273  case X86::BLSIC64rr:
1274  case X86::BLSMSK32rr:
1275  case X86::BLSMSK64rr:
1276  case X86::BLSR32rr:
1277  case X86::BLSR64rr:
1278  case X86::TZMSK32rr:
1279  case X86::TZMSK64rr:
1280 
1281  // Bit extracting and clearing instructions should execute in constant time,
1282  // and set flags.
1283  case X86::BEXTR32rr:
1284  case X86::BEXTR64rr:
1285  case X86::BEXTRI32ri:
1286  case X86::BEXTRI64ri:
1287  case X86::BZHI32rr:
1288  case X86::BZHI64rr:
1289 
1290  // Shift and rotate.
1291  case X86::ROL8r1: case X86::ROL16r1: case X86::ROL32r1: case X86::ROL64r1:
1292  case X86::ROL8rCL: case X86::ROL16rCL: case X86::ROL32rCL: case X86::ROL64rCL:
1293  case X86::ROL8ri: case X86::ROL16ri: case X86::ROL32ri: case X86::ROL64ri:
1294  case X86::ROR8r1: case X86::ROR16r1: case X86::ROR32r1: case X86::ROR64r1:
1295  case X86::ROR8rCL: case X86::ROR16rCL: case X86::ROR32rCL: case X86::ROR64rCL:
1296  case X86::ROR8ri: case X86::ROR16ri: case X86::ROR32ri: case X86::ROR64ri:
1297  case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1: case X86::SAR64r1:
1298  case X86::SAR8rCL: case X86::SAR16rCL: case X86::SAR32rCL: case X86::SAR64rCL:
1299  case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri: case X86::SAR64ri:
1300  case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1: case X86::SHL64r1:
1301  case X86::SHL8rCL: case X86::SHL16rCL: case X86::SHL32rCL: case X86::SHL64rCL:
1302  case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri: case X86::SHL64ri:
1303  case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1: case X86::SHR64r1:
1304  case X86::SHR8rCL: case X86::SHR16rCL: case X86::SHR32rCL: case X86::SHR64rCL:
1305  case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri: case X86::SHR64ri:
1306  case X86::SHLD16rrCL: case X86::SHLD32rrCL: case X86::SHLD64rrCL:
1307  case X86::SHLD16rri8: case X86::SHLD32rri8: case X86::SHLD64rri8:
1308  case X86::SHRD16rrCL: case X86::SHRD32rrCL: case X86::SHRD64rrCL:
1309  case X86::SHRD16rri8: case X86::SHRD32rri8: case X86::SHRD64rri8:
1310 
1311  // Basic arithmetic is constant time on the input but does set flags.
1312  case X86::ADC8rr: case X86::ADC8ri:
1313  case X86::ADC16rr: case X86::ADC16ri: case X86::ADC16ri8:
1314  case X86::ADC32rr: case X86::ADC32ri: case X86::ADC32ri8:
1315  case X86::ADC64rr: case X86::ADC64ri8: case X86::ADC64ri32:
1316  case X86::ADD8rr: case X86::ADD8ri:
1317  case X86::ADD16rr: case X86::ADD16ri: case X86::ADD16ri8:
1318  case X86::ADD32rr: case X86::ADD32ri: case X86::ADD32ri8:
1319  case X86::ADD64rr: case X86::ADD64ri8: case X86::ADD64ri32:
1320  case X86::AND8rr: case X86::AND8ri:
1321  case X86::AND16rr: case X86::AND16ri: case X86::AND16ri8:
1322  case X86::AND32rr: case X86::AND32ri: case X86::AND32ri8:
1323  case X86::AND64rr: case X86::AND64ri8: case X86::AND64ri32:
1324  case X86::OR8rr: case X86::OR8ri:
1325  case X86::OR16rr: case X86::OR16ri: case X86::OR16ri8:
1326  case X86::OR32rr: case X86::OR32ri: case X86::OR32ri8:
1327  case X86::OR64rr: case X86::OR64ri8: case X86::OR64ri32:
1328  case X86::SBB8rr: case X86::SBB8ri:
1329  case X86::SBB16rr: case X86::SBB16ri: case X86::SBB16ri8:
1330  case X86::SBB32rr: case X86::SBB32ri: case X86::SBB32ri8:
1331  case X86::SBB64rr: case X86::SBB64ri8: case X86::SBB64ri32:
1332  case X86::SUB8rr: case X86::SUB8ri:
1333  case X86::SUB16rr: case X86::SUB16ri: case X86::SUB16ri8:
1334  case X86::SUB32rr: case X86::SUB32ri: case X86::SUB32ri8:
1335  case X86::SUB64rr: case X86::SUB64ri8: case X86::SUB64ri32:
1336  case X86::XOR8rr: case X86::XOR8ri:
1337  case X86::XOR16rr: case X86::XOR16ri: case X86::XOR16ri8:
1338  case X86::XOR32rr: case X86::XOR32ri: case X86::XOR32ri8:
1339  case X86::XOR64rr: case X86::XOR64ri8: case X86::XOR64ri32:
1340  // Arithmetic with just 32-bit and 64-bit variants and no immediates.
1341  case X86::ADCX32rr: case X86::ADCX64rr:
1342  case X86::ADOX32rr: case X86::ADOX64rr:
1343  case X86::ANDN32rr: case X86::ANDN64rr:
1344  // Unary arithmetic operations.
1345  case X86::DEC8r: case X86::DEC16r: case X86::DEC32r: case X86::DEC64r:
1346  case X86::INC8r: case X86::INC16r: case X86::INC32r: case X86::INC64r:
1347  case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r:
1348  // Check whether the EFLAGS implicit-def is dead. We assume that this will
1349  // always find the implicit-def because this code should only be reached
1350  // for instructions that do in fact implicitly def this.
1351  if (!MI.findRegisterDefOperand(X86::EFLAGS)->isDead()) {
1352  // If we would clobber EFLAGS that are used, just bail for now.
1353  LLVM_DEBUG(dbgs() << " Unable to harden post-load due to EFLAGS: ";
1354  MI.dump(); dbgs() << "\n");
1355  return false;
1356  }
1357 
1358  // Otherwise, fallthrough to handle these the same as instructions that
1359  // don't set EFLAGS.
1361 
1362  // Unlike other arithmetic, NOT doesn't set EFLAGS.
1363  case X86::NOT8r: case X86::NOT16r: case X86::NOT32r: case X86::NOT64r:
1364 
1365  // Various move instructions used to zero or sign extend things. Note that we
1366  // intentionally don't support the _NOREX variants as we can't handle that
1367  // register constraint anyways.
1368  case X86::MOVSX16rr8:
1369  case X86::MOVSX32rr8: case X86::MOVSX32rr16:
1370  case X86::MOVSX64rr8: case X86::MOVSX64rr16: case X86::MOVSX64rr32:
1371  case X86::MOVZX16rr8:
1372  case X86::MOVZX32rr8: case X86::MOVZX32rr16:
1373  case X86::MOVZX64rr8: case X86::MOVZX64rr16:
1374  case X86::MOV32rr:
1375 
1376  // Arithmetic instructions that are both constant time and don't set flags.
1377  case X86::RORX32ri:
1378  case X86::RORX64ri:
1379  case X86::SARX32rr:
1380  case X86::SARX64rr:
1381  case X86::SHLX32rr:
1382  case X86::SHLX64rr:
1383  case X86::SHRX32rr:
1384  case X86::SHRX64rr:
1385 
1386  // LEA doesn't actually access memory, and its arithmetic is constant time.
1387  case X86::LEA16r:
1388  case X86::LEA32r:
1389  case X86::LEA64_32r:
1390  case X86::LEA64r:
1391  return true;
1392  }
1393 }
1394 
1395 /// Returns true if the instruction has no behavior (specified or otherwise)
1396 /// that is based on the value loaded from memory or the value of any
1397 /// non-address register operands.
1398 ///
1399 /// For example, if the latency of the instruction is dependent on the
1400 /// particular bits set in any of the registers *or* any of the bits loaded from
1401 /// memory.
1402 ///
1403 /// A classical example of something that is inherently not data invariant is an
1404 /// indirect jump -- the destination is loaded into icache based on the bits set
1405 /// in the jump destination register.
1406 ///
1407 /// FIXME: This should become part of our instruction tables.
1409  switch (MI.getOpcode()) {
1410  default:
1411  // By default, assume that the load will immediately leak.
1412  return false;
1413 
1414  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
1415  // However, they set flags and are perhaps the most surprisingly constant
1416  // time operations so we call them out here separately.
1417  case X86::IMUL16rm:
1418  case X86::IMUL16rmi8:
1419  case X86::IMUL16rmi:
1420  case X86::IMUL32rm:
1421  case X86::IMUL32rmi8:
1422  case X86::IMUL32rmi:
1423  case X86::IMUL64rm:
1424  case X86::IMUL64rmi32:
1425  case X86::IMUL64rmi8:
1426 
1427  // Bit scanning and counting instructions that are somewhat surprisingly
1428  // constant time as they scan across bits and do other fairly complex
1429  // operations like popcnt, but are believed to be constant time on x86.
1430  // However, these set flags.
1431  case X86::BSF16rm:
1432  case X86::BSF32rm:
1433  case X86::BSF64rm:
1434  case X86::BSR16rm:
1435  case X86::BSR32rm:
1436  case X86::BSR64rm:
1437  case X86::LZCNT16rm:
1438  case X86::LZCNT32rm:
1439  case X86::LZCNT64rm:
1440  case X86::POPCNT16rm:
1441  case X86::POPCNT32rm:
1442  case X86::POPCNT64rm:
1443  case X86::TZCNT16rm:
1444  case X86::TZCNT32rm:
1445  case X86::TZCNT64rm:
1446 
1447  // Bit manipulation instructions are effectively combinations of basic
1448  // arithmetic ops, and should still execute in constant time. These also
1449  // set flags.
1450  case X86::BLCFILL32rm:
1451  case X86::BLCFILL64rm:
1452  case X86::BLCI32rm:
1453  case X86::BLCI64rm:
1454  case X86::BLCIC32rm:
1455  case X86::BLCIC64rm:
1456  case X86::BLCMSK32rm:
1457  case X86::BLCMSK64rm:
1458  case X86::BLCS32rm:
1459  case X86::BLCS64rm:
1460  case X86::BLSFILL32rm:
1461  case X86::BLSFILL64rm:
1462  case X86::BLSI32rm:
1463  case X86::BLSI64rm:
1464  case X86::BLSIC32rm:
1465  case X86::BLSIC64rm:
1466  case X86::BLSMSK32rm:
1467  case X86::BLSMSK64rm:
1468  case X86::BLSR32rm:
1469  case X86::BLSR64rm:
1470  case X86::TZMSK32rm:
1471  case X86::TZMSK64rm:
1472 
1473  // Bit extracting and clearing instructions should execute in constant time,
1474  // and set flags.
1475  case X86::BEXTR32rm:
1476  case X86::BEXTR64rm:
1477  case X86::BEXTRI32mi:
1478  case X86::BEXTRI64mi:
1479  case X86::BZHI32rm:
1480  case X86::BZHI64rm:
1481 
1482  // Basic arithmetic is constant time on the input but does set flags.
1483  case X86::ADC8rm:
1484  case X86::ADC16rm:
1485  case X86::ADC32rm:
1486  case X86::ADC64rm:
1487  case X86::ADCX32rm:
1488  case X86::ADCX64rm:
1489  case X86::ADD8rm:
1490  case X86::ADD16rm:
1491  case X86::ADD32rm:
1492  case X86::ADD64rm:
1493  case X86::ADOX32rm:
1494  case X86::ADOX64rm:
1495  case X86::AND8rm:
1496  case X86::AND16rm:
1497  case X86::AND32rm:
1498  case X86::AND64rm:
1499  case X86::ANDN32rm:
1500  case X86::ANDN64rm:
1501  case X86::OR8rm:
1502  case X86::OR16rm:
1503  case X86::OR32rm:
1504  case X86::OR64rm:
1505  case X86::SBB8rm:
1506  case X86::SBB16rm:
1507  case X86::SBB32rm:
1508  case X86::SBB64rm:
1509  case X86::SUB8rm:
1510  case X86::SUB16rm:
1511  case X86::SUB32rm:
1512  case X86::SUB64rm:
1513  case X86::XOR8rm:
1514  case X86::XOR16rm:
1515  case X86::XOR32rm:
1516  case X86::XOR64rm:
1517  // Check whether the EFLAGS implicit-def is dead. We assume that this will
1518  // always find the implicit-def because this code should only be reached
1519  // for instructions that do in fact implicitly def this.
1520  if (!MI.findRegisterDefOperand(X86::EFLAGS)->isDead()) {
1521  // If we would clobber EFLAGS that are used, just bail for now.
1522  LLVM_DEBUG(dbgs() << " Unable to harden post-load due to EFLAGS: ";
1523  MI.dump(); dbgs() << "\n");
1524  return false;
1525  }
1526 
1527  // Otherwise, fallthrough to handle these the same as instructions that
1528  // don't set EFLAGS.
1530 
1531  // Integer multiply w/o affecting flags is still believed to be constant
1532  // time on x86. Called out separately as this is among the most surprising
1533  // instructions to exhibit that behavior.
1534  case X86::MULX32rm:
1535  case X86::MULX64rm:
1536 
1537  // Arithmetic instructions that are both constant time and don't set flags.
1538  case X86::RORX32mi:
1539  case X86::RORX64mi:
1540  case X86::SARX32rm:
1541  case X86::SARX64rm:
1542  case X86::SHLX32rm:
1543  case X86::SHLX64rm:
1544  case X86::SHRX32rm:
1545  case X86::SHRX64rm:
1546 
1547  // Conversions are believed to be constant time and don't set flags.
1548  case X86::CVTTSD2SI64rm: case X86::VCVTTSD2SI64rm: case X86::VCVTTSD2SI64Zrm:
1549  case X86::CVTTSD2SIrm: case X86::VCVTTSD2SIrm: case X86::VCVTTSD2SIZrm:
1550  case X86::CVTTSS2SI64rm: case X86::VCVTTSS2SI64rm: case X86::VCVTTSS2SI64Zrm:
1551  case X86::CVTTSS2SIrm: case X86::VCVTTSS2SIrm: case X86::VCVTTSS2SIZrm:
1552  case X86::CVTSI2SDrm: case X86::VCVTSI2SDrm: case X86::VCVTSI2SDZrm:
1553  case X86::CVTSI2SSrm: case X86::VCVTSI2SSrm: case X86::VCVTSI2SSZrm:
1554  case X86::CVTSI642SDrm: case X86::VCVTSI642SDrm: case X86::VCVTSI642SDZrm:
1555  case X86::CVTSI642SSrm: case X86::VCVTSI642SSrm: case X86::VCVTSI642SSZrm:
1556  case X86::CVTSS2SDrm: case X86::VCVTSS2SDrm: case X86::VCVTSS2SDZrm:
1557  case X86::CVTSD2SSrm: case X86::VCVTSD2SSrm: case X86::VCVTSD2SSZrm:
1558  // AVX512 added unsigned integer conversions.
1559  case X86::VCVTTSD2USI64Zrm:
1560  case X86::VCVTTSD2USIZrm:
1561  case X86::VCVTTSS2USI64Zrm:
1562  case X86::VCVTTSS2USIZrm:
1563  case X86::VCVTUSI2SDZrm:
1564  case X86::VCVTUSI642SDZrm:
1565  case X86::VCVTUSI2SSZrm:
1566  case X86::VCVTUSI642SSZrm:
1567 
1568  // Loads to register don't set flags.
1569  case X86::MOV8rm:
1570  case X86::MOV8rm_NOREX:
1571  case X86::MOV16rm:
1572  case X86::MOV32rm:
1573  case X86::MOV64rm:
1574  case X86::MOVSX16rm8:
1575  case X86::MOVSX32rm16:
1576  case X86::MOVSX32rm8:
1577  case X86::MOVSX32rm8_NOREX:
1578  case X86::MOVSX64rm16:
1579  case X86::MOVSX64rm32:
1580  case X86::MOVSX64rm8:
1581  case X86::MOVZX16rm8:
1582  case X86::MOVZX32rm16:
1583  case X86::MOVZX32rm8:
1584  case X86::MOVZX32rm8_NOREX:
1585  case X86::MOVZX64rm16:
1586  case X86::MOVZX64rm8:
1587  return true;
1588  }
1589 }
1590 
1592  const TargetRegisterInfo &TRI) {
1593  // Check if EFLAGS are alive by seeing if there is a def of them or they
1594  // live-in, and then seeing if that def is in turn used.
1595  for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) {
1596  if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) {
1597  // If the def is dead, then EFLAGS is not live.
1598  if (DefOp->isDead())
1599  return false;
1600 
1601  // Otherwise we've def'ed it, and it is live.
1602  return true;
1603  }
1604  // While at this instruction, also check if we use and kill EFLAGS
1605  // which means it isn't live.
1606  if (MI.killsRegister(X86::EFLAGS, &TRI))
1607  return false;
1608  }
1609 
1610  // If we didn't find anything conclusive (neither definitely alive or
1611  // definitely dead) return whether it lives into the block.
1612  return MBB.isLiveIn(X86::EFLAGS);
1613 }
1614 
1615 /// Trace the predicate state through each of the blocks in the function,
1616 /// hardening everything necessary along the way.
1617 ///
1618 /// We call this routine once the initial predicate state has been established
1619 /// for each basic block in the function in the SSA updater. This routine traces
1620 /// it through the instructions within each basic block, and for non-returning
1621 /// blocks informs the SSA updater about the final state that lives out of the
1622 /// block. Along the way, it hardens any vulnerable instruction using the
1623 /// currently valid predicate state. We have to do these two things together
1624 /// because the SSA updater only works across blocks. Within a block, we track
1625 /// the current predicate state directly and update it as it changes.
1626 ///
1627 /// This operates in two passes over each block. First, we analyze the loads in
1628 /// the block to determine which strategy will be used to harden them: hardening
1629 /// the address or hardening the loaded value when loaded into a register
1630 /// amenable to hardening. We have to process these first because the two
1631 /// strategies may interact -- later hardening may change what strategy we wish
1632 /// to use. We also will analyze data dependencies between loads and avoid
1633 /// hardening those loads that are data dependent on a load with a hardened
1634 /// address. We also skip hardening loads already behind an LFENCE as that is
1635 /// sufficient to harden them against misspeculation.
1636 ///
1637 /// Second, we actively trace the predicate state through the block, applying
1638 /// the hardening steps we determined necessary in the first pass as we go.
1639 ///
1640 /// These two passes are applied to each basic block. We operate one block at a
1641 /// time to simplify reasoning about reachability and sequencing.
1642 void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(
1643  MachineFunction &MF) {
1644  SmallPtrSet<MachineInstr *, 16> HardenPostLoad;
1645  SmallPtrSet<MachineInstr *, 16> HardenLoadAddr;
1646 
1647  SmallSet<unsigned, 16> HardenedAddrRegs;
1648 
1649  SmallDenseMap<unsigned, unsigned, 32> AddrRegToHardenedReg;
1650 
1651  // Track the set of load-dependent registers through the basic block. Because
1652  // the values of these registers have an existing data dependency on a loaded
1653  // value which we would have checked, we can omit any checks on them.
1654  SparseBitVector<> LoadDepRegs;
1655 
1656  for (MachineBasicBlock &MBB : MF) {
1657  // The first pass over the block: collect all the loads which can have their
1658  // loaded value hardened and all the loads that instead need their address
1659  // hardened. During this walk we propagate load dependence for address
1660  // hardened loads and also look for LFENCE to stop hardening wherever
1661  // possible. When deciding whether or not to harden the loaded value or not,
1662  // we check to see if any registers used in the address will have been
1663  // hardened at this point and if so, harden any remaining address registers
1664  // as that often successfully re-uses hardened addresses and minimizes
1665  // instructions.
1666  //
1667  // FIXME: We should consider an aggressive mode where we continue to keep as
1668  // many loads value hardened even when some address register hardening would
1669  // be free (due to reuse).
1670  //
1671  // Note that we only need this pass if we are actually hardening loads.
1672  if (HardenLoads)
1673  for (MachineInstr &MI : MBB) {
1674  // We naively assume that all def'ed registers of an instruction have
1675  // a data dependency on all of their operands.
1676  // FIXME: Do a more careful analysis of x86 to build a conservative
1677  // model here.
1678  if (llvm::any_of(MI.uses(), [&](MachineOperand &Op) {
1679  return Op.isReg() && LoadDepRegs.test(Op.getReg());
1680  }))
1681  for (MachineOperand &Def : MI.defs())
1682  if (Def.isReg())
1683  LoadDepRegs.set(Def.getReg());
1684 
1685  // Both Intel and AMD are guiding that they will change the semantics of
1686  // LFENCE to be a speculation barrier, so if we see an LFENCE, there is
1687  // no more need to guard things in this block.
1688  if (MI.getOpcode() == X86::LFENCE)
1689  break;
1690 
1691  // If this instruction cannot load, nothing to do.
1692  if (!MI.mayLoad())
1693  continue;
1694 
1695  // Some instructions which "load" are trivially safe or unimportant.
1696  if (MI.getOpcode() == X86::MFENCE)
1697  continue;
1698 
1699  // Extract the memory operand information about this instruction.
1700  // FIXME: This doesn't handle loading pseudo instructions which we often
1701  // could handle with similarly generic logic. We probably need to add an
1702  // MI-layer routine similar to the MC-layer one we use here which maps
1703  // pseudos much like this maps real instructions.
1704  const MCInstrDesc &Desc = MI.getDesc();
1705  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
1706  if (MemRefBeginIdx < 0) {
1707  LLVM_DEBUG(dbgs()
1708  << "WARNING: unable to harden loading instruction: ";
1709  MI.dump());
1710  continue;
1711  }
1712 
1713  MemRefBeginIdx += X86II::getOperandBias(Desc);
1714 
1715  MachineOperand &BaseMO =
1716  MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
1717  MachineOperand &IndexMO =
1718  MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
1719 
1720  // If we have at least one (non-frame-index, non-RIP) register operand,
1721  // and neither operand is load-dependent, we need to check the load.
1722  unsigned BaseReg = 0, IndexReg = 0;
1723  if (!BaseMO.isFI() && BaseMO.getReg() != X86::RIP &&
1724  BaseMO.getReg() != X86::NoRegister)
1725  BaseReg = BaseMO.getReg();
1726  if (IndexMO.getReg() != X86::NoRegister)
1727  IndexReg = IndexMO.getReg();
1728 
1729  if (!BaseReg && !IndexReg)
1730  // No register operands!
1731  continue;
1732 
1733  // If any register operand is dependent, this load is dependent and we
1734  // needn't check it.
1735  // FIXME: Is this true in the case where we are hardening loads after
1736  // they complete? Unclear, need to investigate.
1737  if ((BaseReg && LoadDepRegs.test(BaseReg)) ||
1738  (IndexReg && LoadDepRegs.test(IndexReg)))
1739  continue;
1740 
1741  // If post-load hardening is enabled, this load is compatible with
1742  // post-load hardening, and we aren't already going to harden one of the
1743  // address registers, queue it up to be hardened post-load. Notably,
1744  // even once hardened this won't introduce a useful dependency that
1745  // could prune out subsequent loads.
1747  MI.getDesc().getNumDefs() == 1 && MI.getOperand(0).isReg() &&
1748  canHardenRegister(MI.getOperand(0).getReg()) &&
1749  !HardenedAddrRegs.count(BaseReg) &&
1750  !HardenedAddrRegs.count(IndexReg)) {
1751  HardenPostLoad.insert(&MI);
1752  HardenedAddrRegs.insert(MI.getOperand(0).getReg());
1753  continue;
1754  }
1755 
1756  // Record this instruction for address hardening and record its register
1757  // operands as being address-hardened.
1758  HardenLoadAddr.insert(&MI);
1759  if (BaseReg)
1760  HardenedAddrRegs.insert(BaseReg);
1761  if (IndexReg)
1762  HardenedAddrRegs.insert(IndexReg);
1763 
1764  for (MachineOperand &Def : MI.defs())
1765  if (Def.isReg())
1766  LoadDepRegs.set(Def.getReg());
1767  }
1768 
1769  // Now re-walk the instructions in the basic block, and apply whichever
1770  // hardening strategy we have elected. Note that we do this in a second
1771  // pass specifically so that we have the complete set of instructions for
1772  // which we will do post-load hardening and can defer it in certain
1773  // circumstances.
1774  for (MachineInstr &MI : MBB) {
1775  if (HardenLoads) {
1776  // We cannot both require hardening the def of a load and its address.
1777  assert(!(HardenLoadAddr.count(&MI) && HardenPostLoad.count(&MI)) &&
1778  "Requested to harden both the address and def of a load!");
1779 
1780  // Check if this is a load whose address needs to be hardened.
1781  if (HardenLoadAddr.erase(&MI)) {
1782  const MCInstrDesc &Desc = MI.getDesc();
1783  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
1784  assert(MemRefBeginIdx >= 0 && "Cannot have an invalid index here!");
1785 
1786  MemRefBeginIdx += X86II::getOperandBias(Desc);
1787 
1788  MachineOperand &BaseMO =
1789  MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
1790  MachineOperand &IndexMO =
1791  MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
1792  hardenLoadAddr(MI, BaseMO, IndexMO, AddrRegToHardenedReg);
1793  continue;
1794  }
1795 
1796  // Test if this instruction is one of our post load instructions (and
1797  // remove it from the set if so).
1798  if (HardenPostLoad.erase(&MI)) {
1799  assert(!MI.isCall() && "Must not try to post-load harden a call!");
1800 
1801  // If this is a data-invariant load, we want to try and sink any
1802  // hardening as far as possible.
1803  if (isDataInvariantLoad(MI)) {
1804  // Sink the instruction we'll need to harden as far as we can down
1805  // the graph.
1806  MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad);
1807 
1808  // If we managed to sink this instruction, update everything so we
1809  // harden that instruction when we reach it in the instruction
1810  // sequence.
1811  if (SunkMI != &MI) {
1812  // If in sinking there was no instruction needing to be hardened,
1813  // we're done.
1814  if (!SunkMI)
1815  continue;
1816 
1817  // Otherwise, add this to the set of defs we harden.
1818  HardenPostLoad.insert(SunkMI);
1819  continue;
1820  }
1821  }
1822 
1823  unsigned HardenedReg = hardenPostLoad(MI);
1824 
1825  // Mark the resulting hardened register as such so we don't re-harden.
1826  AddrRegToHardenedReg[HardenedReg] = HardenedReg;
1827 
1828  continue;
1829  }
1830 
1831  // Check for an indirect call or branch that may need its input hardened
1832  // even if we couldn't find the specific load used, or were able to
1833  // avoid hardening it for some reason. Note that here we cannot break
1834  // out afterward as we may still need to handle any call aspect of this
1835  // instruction.
1836  if ((MI.isCall() || MI.isBranch()) && HardenIndirectCallsAndJumps)
1837  hardenIndirectCallOrJumpInstr(MI, AddrRegToHardenedReg);
1838  }
1839 
1840  // After we finish hardening loads we handle interprocedural hardening if
1841  // enabled and relevant for this instruction.
1843  continue;
1844  if (!MI.isCall() && !MI.isReturn())
1845  continue;
1846 
1847  // If this is a direct return (IE, not a tail call) just directly harden
1848  // it.
1849  if (MI.isReturn() && !MI.isCall()) {
1850  hardenReturnInstr(MI);
1851  continue;
1852  }
1853 
1854  // Otherwise we have a call. We need to handle transferring the predicate
1855  // state into a call and recovering it after the call returns (unless this
1856  // is a tail call).
1857  assert(MI.isCall() && "Should only reach here for calls!");
1858  tracePredStateThroughCall(MI);
1859  }
1860 
1861  HardenPostLoad.clear();
1862  HardenLoadAddr.clear();
1863  HardenedAddrRegs.clear();
1864  AddrRegToHardenedReg.clear();
1865 
1866  // Currently, we only track data-dependent loads within a basic block.
1867  // FIXME: We should see if this is necessary or if we could be more
1868  // aggressive here without opening up attack avenues.
1869  LoadDepRegs.clear();
1870  }
1871 }
1872 
1873 /// Save EFLAGS into the returned GPR. This can in turn be restored with
1874 /// `restoreEFLAGS`.
1875 ///
1876 /// Note that LLVM can only lower very simple patterns of saved and restored
1877 /// EFLAGS registers. The restore should always be within the same basic block
1878 /// as the save so that no PHI nodes are inserted.
1879 unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS(
1881  DebugLoc Loc) {
1882  // FIXME: Hard coding this to a 32-bit register class seems weird, but matches
1883  // what instruction selection does.
1884  unsigned Reg = MRI->createVirtualRegister(&X86::GR32RegClass);
1885  // We directly copy the FLAGS register and rely on later lowering to clean
1886  // this up into the appropriate setCC instructions.
1887  BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), Reg).addReg(X86::EFLAGS);
1888  ++NumInstsInserted;
1889  return Reg;
1890 }
1891 
1892 /// Restore EFLAGS from the provided GPR. This should be produced by
1893 /// `saveEFLAGS`.
1894 ///
1895 /// This must be done within the same basic block as the save in order to
1896 /// reliably lower.
1897 void X86SpeculativeLoadHardeningPass::restoreEFLAGS(
1899  unsigned Reg) {
1900  BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), X86::EFLAGS).addReg(Reg);
1901  ++NumInstsInserted;
1902 }
1903 
1904 /// Takes the current predicate state (in a register) and merges it into the
1905 /// stack pointer. The state is essentially a single bit, but we merge this in
1906 /// a way that won't form non-canonical pointers and also will be preserved
1907 /// across normal stack adjustments.
1908 void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(
1910  unsigned PredStateReg) {
1911  unsigned TmpReg = MRI->createVirtualRegister(PS->RC);
1912  // FIXME: This hard codes a shift distance based on the number of bits needed
1913  // to stay canonical on 64-bit. We should compute this somehow and support
1914  // 32-bit as part of that.
1915  auto ShiftI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHL64ri), TmpReg)
1916  .addReg(PredStateReg, RegState::Kill)
1917  .addImm(47);
1918  ShiftI->addRegisterDead(X86::EFLAGS, TRI);
1919  ++NumInstsInserted;
1920  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), X86::RSP)
1921  .addReg(X86::RSP)
1922  .addReg(TmpReg, RegState::Kill);
1923  OrI->addRegisterDead(X86::EFLAGS, TRI);
1924  ++NumInstsInserted;
1925 }
1926 
1927 /// Extracts the predicate state stored in the high bits of the stack pointer.
1928 unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP(
1930  DebugLoc Loc) {
1931  unsigned PredStateReg = MRI->createVirtualRegister(PS->RC);
1932  unsigned TmpReg = MRI->createVirtualRegister(PS->RC);
1933 
1934  // We know that the stack pointer will have any preserved predicate state in
1935  // its high bit. We just want to smear this across the other bits. Turns out,
1936  // this is exactly what an arithmetic right shift does.
1937  BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), TmpReg)
1938  .addReg(X86::RSP);
1939  auto ShiftI =
1940  BuildMI(MBB, InsertPt, Loc, TII->get(X86::SAR64ri), PredStateReg)
1941  .addReg(TmpReg, RegState::Kill)
1942  .addImm(TRI->getRegSizeInBits(*PS->RC) - 1);
1943  ShiftI->addRegisterDead(X86::EFLAGS, TRI);
1944  ++NumInstsInserted;
1945 
1946  return PredStateReg;
1947 }
1948 
1949 void X86SpeculativeLoadHardeningPass::hardenLoadAddr(
1950  MachineInstr &MI, MachineOperand &BaseMO, MachineOperand &IndexMO,
1951  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
1952  MachineBasicBlock &MBB = *MI.getParent();
1953  DebugLoc Loc = MI.getDebugLoc();
1954 
1955  // Check if EFLAGS are alive by seeing if there is a def of them or they
1956  // live-in, and then seeing if that def is in turn used.
1957  bool EFLAGSLive = isEFLAGSLive(MBB, MI.getIterator(), *TRI);
1958 
1959  SmallVector<MachineOperand *, 2> HardenOpRegs;
1960 
1961  if (BaseMO.isFI()) {
1962  // A frame index is never a dynamically controllable load, so only
1963  // harden it if we're covering fixed address loads as well.
1964  LLVM_DEBUG(
1965  dbgs() << " Skipping hardening base of explicit stack frame load: ";
1966  MI.dump(); dbgs() << "\n");
1967  } else if (BaseMO.getReg() == X86::RIP ||
1968  BaseMO.getReg() == X86::NoRegister) {
1969  // For both RIP-relative addressed loads or absolute loads, we cannot
1970  // meaningfully harden them because the address being loaded has no
1971  // dynamic component.
1972  //
1973  // FIXME: When using a segment base (like TLS does) we end up with the
1974  // dynamic address being the base plus -1 because we can't mutate the
1975  // segment register here. This allows the signed 32-bit offset to point at
1976  // valid segment-relative addresses and load them successfully.
1977  LLVM_DEBUG(
1978  dbgs() << " Cannot harden base of "
1979  << (BaseMO.getReg() == X86::RIP ? "RIP-relative" : "no-base")
1980  << " address in a load!");
1981  } else {
1982  assert(BaseMO.isReg() &&
1983  "Only allowed to have a frame index or register base.");
1984  HardenOpRegs.push_back(&BaseMO);
1985  }
1986 
1987  if (IndexMO.getReg() != X86::NoRegister &&
1988  (HardenOpRegs.empty() ||
1989  HardenOpRegs.front()->getReg() != IndexMO.getReg()))
1990  HardenOpRegs.push_back(&IndexMO);
1991 
1992  assert((HardenOpRegs.size() == 1 || HardenOpRegs.size() == 2) &&
1993  "Should have exactly one or two registers to harden!");
1994  assert((HardenOpRegs.size() == 1 ||
1995  HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&
1996  "Should not have two of the same registers!");
1997 
1998  // Remove any registers that have alreaded been checked.
1999  llvm::erase_if(HardenOpRegs, [&](MachineOperand *Op) {
2000  // See if this operand's register has already been checked.
2001  auto It = AddrRegToHardenedReg.find(Op->getReg());
2002  if (It == AddrRegToHardenedReg.end())
2003  // Not checked, so retain this one.
2004  return false;
2005 
2006  // Otherwise, we can directly update this operand and remove it.
2007  Op->setReg(It->second);
2008  return true;
2009  });
2010  // If there are none left, we're done.
2011  if (HardenOpRegs.empty())
2012  return;
2013 
2014  // Compute the current predicate state.
2015  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2016 
2017  auto InsertPt = MI.getIterator();
2018 
2019  // If EFLAGS are live and we don't have access to instructions that avoid
2020  // clobbering EFLAGS we need to save and restore them. This in turn makes
2021  // the EFLAGS no longer live.
2022  unsigned FlagsReg = 0;
2023  if (EFLAGSLive && !Subtarget->hasBMI2()) {
2024  EFLAGSLive = false;
2025  FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
2026  }
2027 
2028  for (MachineOperand *Op : HardenOpRegs) {
2029  unsigned OpReg = Op->getReg();
2030  auto *OpRC = MRI->getRegClass(OpReg);
2031  unsigned TmpReg = MRI->createVirtualRegister(OpRC);
2032 
2033  // If this is a vector register, we'll need somewhat custom logic to handle
2034  // hardening it.
2035  if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) ||
2036  OpRC->hasSuperClassEq(&X86::VR256RegClass))) {
2037  assert(Subtarget->hasAVX2() && "AVX2-specific register classes!");
2038  bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);
2039 
2040  // Move our state into a vector register.
2041  // FIXME: We could skip this at the cost of longer encodings with AVX-512
2042  // but that doesn't seem likely worth it.
2043  unsigned VStateReg = MRI->createVirtualRegister(&X86::VR128RegClass);
2044  auto MovI =
2045  BuildMI(MBB, InsertPt, Loc, TII->get(X86::VMOV64toPQIrr), VStateReg)
2046  .addReg(StateReg);
2047  (void)MovI;
2048  ++NumInstsInserted;
2049  LLVM_DEBUG(dbgs() << " Inserting mov: "; MovI->dump(); dbgs() << "\n");
2050 
2051  // Broadcast it across the vector register.
2052  unsigned VBStateReg = MRI->createVirtualRegister(OpRC);
2053  auto BroadcastI = BuildMI(MBB, InsertPt, Loc,
2054  TII->get(Is128Bit ? X86::VPBROADCASTQrr
2055  : X86::VPBROADCASTQYrr),
2056  VBStateReg)
2057  .addReg(VStateReg);
2058  (void)BroadcastI;
2059  ++NumInstsInserted;
2060  LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();
2061  dbgs() << "\n");
2062 
2063  // Merge our potential poison state into the value with a vector or.
2064  auto OrI =
2065  BuildMI(MBB, InsertPt, Loc,
2066  TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg)
2067  .addReg(VBStateReg)
2068  .addReg(OpReg);
2069  (void)OrI;
2070  ++NumInstsInserted;
2071  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2072  } else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||
2073  OpRC->hasSuperClassEq(&X86::VR256XRegClass) ||
2074  OpRC->hasSuperClassEq(&X86::VR512RegClass)) {
2075  assert(Subtarget->hasAVX512() && "AVX512-specific register classes!");
2076  bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);
2077  bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);
2078  if (Is128Bit || Is256Bit)
2079  assert(Subtarget->hasVLX() && "AVX512VL-specific register classes!");
2080 
2081  // Broadcast our state into a vector register.
2082  unsigned VStateReg = MRI->createVirtualRegister(OpRC);
2083  unsigned BroadcastOp =
2084  Is128Bit ? X86::VPBROADCASTQrZ128r
2085  : Is256Bit ? X86::VPBROADCASTQrZ256r : X86::VPBROADCASTQrZr;
2086  auto BroadcastI =
2087  BuildMI(MBB, InsertPt, Loc, TII->get(BroadcastOp), VStateReg)
2088  .addReg(StateReg);
2089  (void)BroadcastI;
2090  ++NumInstsInserted;
2091  LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();
2092  dbgs() << "\n");
2093 
2094  // Merge our potential poison state into the value with a vector or.
2095  unsigned OrOp = Is128Bit ? X86::VPORQZ128rr
2096  : Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr;
2097  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOp), TmpReg)
2098  .addReg(VStateReg)
2099  .addReg(OpReg);
2100  (void)OrI;
2101  ++NumInstsInserted;
2102  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2103  } else {
2104  // FIXME: Need to support GR32 here for 32-bit code.
2105  assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&
2106  "Not a supported register class for address hardening!");
2107 
2108  if (!EFLAGSLive) {
2109  // Merge our potential poison state into the value with an or.
2110  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), TmpReg)
2111  .addReg(StateReg)
2112  .addReg(OpReg);
2113  OrI->addRegisterDead(X86::EFLAGS, TRI);
2114  ++NumInstsInserted;
2115  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2116  } else {
2117  // We need to avoid touching EFLAGS so shift out all but the least
2118  // significant bit using the instruction that doesn't update flags.
2119  auto ShiftI =
2120  BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHRX64rr), TmpReg)
2121  .addReg(OpReg)
2122  .addReg(StateReg);
2123  (void)ShiftI;
2124  ++NumInstsInserted;
2125  LLVM_DEBUG(dbgs() << " Inserting shrx: "; ShiftI->dump();
2126  dbgs() << "\n");
2127  }
2128  }
2129 
2130  // Record this register as checked and update the operand.
2131  assert(!AddrRegToHardenedReg.count(Op->getReg()) &&
2132  "Should not have checked this register yet!");
2133  AddrRegToHardenedReg[Op->getReg()] = TmpReg;
2134  Op->setReg(TmpReg);
2135  ++NumAddrRegsHardened;
2136  }
2137 
2138  // And restore the flags if needed.
2139  if (FlagsReg)
2140  restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
2141 }
2142 
2143 MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(
2144  MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) {
2145  assert(isDataInvariantLoad(InitialMI) &&
2146  "Cannot get here with a non-invariant load!");
2147 
2148  // See if we can sink hardening the loaded value.
2149  auto SinkCheckToSingleUse =
2151  unsigned DefReg = MI.getOperand(0).getReg();
2152 
2153  // We need to find a single use which we can sink the check. We can
2154  // primarily do this because many uses may already end up checked on their
2155  // own.
2156  MachineInstr *SingleUseMI = nullptr;
2157  for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) {
2158  // If we're already going to harden this use, it is data invariant and
2159  // within our block.
2160  if (HardenedInstrs.count(&UseMI)) {
2161  if (!isDataInvariantLoad(UseMI)) {
2162  // If we've already decided to harden a non-load, we must have sunk
2163  // some other post-load hardened instruction to it and it must itself
2164  // be data-invariant.
2166  "Data variant instruction being hardened!");
2167  continue;
2168  }
2169 
2170  // Otherwise, this is a load and the load component can't be data
2171  // invariant so check how this register is being used.
2172  const MCInstrDesc &Desc = UseMI.getDesc();
2173  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
2174  assert(MemRefBeginIdx >= 0 &&
2175  "Should always have mem references here!");
2176  MemRefBeginIdx += X86II::getOperandBias(Desc);
2177 
2178  MachineOperand &BaseMO =
2179  UseMI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
2180  MachineOperand &IndexMO =
2181  UseMI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
2182  if ((BaseMO.isReg() && BaseMO.getReg() == DefReg) ||
2183  (IndexMO.isReg() && IndexMO.getReg() == DefReg))
2184  // The load uses the register as part of its address making it not
2185  // invariant.
2186  return {};
2187 
2188  continue;
2189  }
2190 
2191  if (SingleUseMI)
2192  // We already have a single use, this would make two. Bail.
2193  return {};
2194 
2195  // If this single use isn't data invariant, isn't in this block, or has
2196  // interfering EFLAGS, we can't sink the hardening to it.
2197  if (!isDataInvariant(UseMI) || UseMI.getParent() != MI.getParent())
2198  return {};
2199 
2200  // If this instruction defines multiple registers bail as we won't harden
2201  // all of them.
2202  if (UseMI.getDesc().getNumDefs() > 1)
2203  return {};
2204 
2205  // If this register isn't a virtual register we can't walk uses of sanely,
2206  // just bail. Also check that its register class is one of the ones we
2207  // can harden.
2208  unsigned UseDefReg = UseMI.getOperand(0).getReg();
2209  if (!TRI->isVirtualRegister(UseDefReg) ||
2210  !canHardenRegister(UseDefReg))
2211  return {};
2212 
2213  SingleUseMI = &UseMI;
2214  }
2215 
2216  // If SingleUseMI is still null, there is no use that needs its own
2217  // checking. Otherwise, it is the single use that needs checking.
2218  return {SingleUseMI};
2219  };
2220 
2221  MachineInstr *MI = &InitialMI;
2222  while (Optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*MI)) {
2223  // Update which MI we're checking now.
2224  MI = *SingleUse;
2225  if (!MI)
2226  break;
2227  }
2228 
2229  return MI;
2230 }
2231 
2232 bool X86SpeculativeLoadHardeningPass::canHardenRegister(unsigned Reg) {
2233  auto *RC = MRI->getRegClass(Reg);
2234  int RegBytes = TRI->getRegSizeInBits(*RC) / 8;
2235  if (RegBytes > 8)
2236  // We don't support post-load hardening of vectors.
2237  return false;
2238 
2239  // If this register class is explicitly constrained to a class that doesn't
2240  // require REX prefix, we may not be able to satisfy that constraint when
2241  // emitting the hardening instructions, so bail out here.
2242  // FIXME: This seems like a pretty lame hack. The way this comes up is when we
2243  // end up both with a NOREX and REX-only register as operands to the hardening
2244  // instructions. It would be better to fix that code to handle this situation
2245  // rather than hack around it in this way.
2246  const TargetRegisterClass *NOREXRegClasses[] = {
2247  &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,
2248  &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};
2249  if (RC == NOREXRegClasses[Log2_32(RegBytes)])
2250  return false;
2251 
2252  const TargetRegisterClass *GPRRegClasses[] = {
2253  &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,
2254  &X86::GR64RegClass};
2255  return RC->hasSuperClassEq(GPRRegClasses[Log2_32(RegBytes)]);
2256 }
2257 
2258 /// Harden a value in a register.
2259 ///
2260 /// This is the low-level logic to fully harden a value sitting in a register
2261 /// against leaking during speculative execution.
2262 ///
2263 /// Unlike hardening an address that is used by a load, this routine is required
2264 /// to hide *all* incoming bits in the register.
2265 ///
2266 /// `Reg` must be a virtual register. Currently, it is required to be a GPR no
2267 /// larger than the predicate state register. FIXME: We should support vector
2268 /// registers here by broadcasting the predicate state.
2269 ///
2270 /// The new, hardened virtual register is returned. It will have the same
2271 /// register class as `Reg`.
2272 unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister(
2273  unsigned Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
2274  DebugLoc Loc) {
2275  assert(canHardenRegister(Reg) && "Cannot harden this register!");
2276  assert(TRI->isVirtualRegister(Reg) && "Cannot harden a physical register!");
2277 
2278  auto *RC = MRI->getRegClass(Reg);
2279  int Bytes = TRI->getRegSizeInBits(*RC) / 8;
2280 
2281  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2282 
2283  // FIXME: Need to teach this about 32-bit mode.
2284  if (Bytes != 8) {
2285  unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};
2286  unsigned SubRegImm = SubRegImms[Log2_32(Bytes)];
2287  unsigned NarrowStateReg = MRI->createVirtualRegister(RC);
2288  BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), NarrowStateReg)
2289  .addReg(StateReg, 0, SubRegImm);
2290  StateReg = NarrowStateReg;
2291  }
2292 
2293  unsigned FlagsReg = 0;
2294  if (isEFLAGSLive(MBB, InsertPt, *TRI))
2295  FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
2296 
2297  unsigned NewReg = MRI->createVirtualRegister(RC);
2298  unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};
2299  unsigned OrOpCode = OrOpCodes[Log2_32(Bytes)];
2300  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOpCode), NewReg)
2301  .addReg(StateReg)
2302  .addReg(Reg);
2303  OrI->addRegisterDead(X86::EFLAGS, TRI);
2304  ++NumInstsInserted;
2305  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2306 
2307  if (FlagsReg)
2308  restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
2309 
2310  return NewReg;
2311 }
2312 
2313 /// Harden a load by hardening the loaded value in the defined register.
2314 ///
2315 /// We can harden a non-leaking load into a register without touching the
2316 /// address by just hiding all of the loaded bits during misspeculation. We use
2317 /// an `or` instruction to do this because we set up our poison value as all
2318 /// ones. And the goal is just for the loaded bits to not be exposed to
2319 /// execution and coercing them to one is sufficient.
2320 ///
2321 /// Returns the newly hardened register.
2322 unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(MachineInstr &MI) {
2323  MachineBasicBlock &MBB = *MI.getParent();
2324  DebugLoc Loc = MI.getDebugLoc();
2325 
2326  auto &DefOp = MI.getOperand(0);
2327  unsigned OldDefReg = DefOp.getReg();
2328  auto *DefRC = MRI->getRegClass(OldDefReg);
2329 
2330  // Because we want to completely replace the uses of this def'ed value with
2331  // the hardened value, create a dedicated new register that will only be used
2332  // to communicate the unhardened value to the hardening.
2333  unsigned UnhardenedReg = MRI->createVirtualRegister(DefRC);
2334  DefOp.setReg(UnhardenedReg);
2335 
2336  // Now harden this register's value, getting a hardened reg that is safe to
2337  // use. Note that we insert the instructions to compute this *after* the
2338  // defining instruction, not before it.
2339  unsigned HardenedReg = hardenValueInRegister(
2340  UnhardenedReg, MBB, std::next(MI.getIterator()), Loc);
2341 
2342  // Finally, replace the old register (which now only has the uses of the
2343  // original def) with the hardened register.
2344  MRI->replaceRegWith(/*FromReg*/ OldDefReg, /*ToReg*/ HardenedReg);
2345 
2346  ++NumPostLoadRegsHardened;
2347  return HardenedReg;
2348 }
2349 
2350 /// Harden a return instruction.
2351 ///
2352 /// Returns implicitly perform a load which we need to harden. Without hardening
2353 /// this load, an attacker my speculatively write over the return address to
2354 /// steer speculation of the return to an attacker controlled address. This is
2355 /// called Spectre v1.1 or Bounds Check Bypass Store (BCBS) and is described in
2356 /// this paper:
2357 /// https://people.csail.mit.edu/vlk/spectre11.pdf
2358 ///
2359 /// We can harden this by introducing an LFENCE that will delay any load of the
2360 /// return address until prior instructions have retired (and thus are not being
2361 /// speculated), or we can harden the address used by the implicit load: the
2362 /// stack pointer.
2363 ///
2364 /// If we are not using an LFENCE, hardening the stack pointer has an additional
2365 /// benefit: it allows us to pass the predicate state accumulated in this
2366 /// function back to the caller. In the absence of a BCBS attack on the return,
2367 /// the caller will typically be resumed and speculatively executed due to the
2368 /// Return Stack Buffer (RSB) prediction which is very accurate and has a high
2369 /// priority. It is possible that some code from the caller will be executed
2370 /// speculatively even during a BCBS-attacked return until the steering takes
2371 /// effect. Whenever this happens, the caller can recover the (poisoned)
2372 /// predicate state from the stack pointer and continue to harden loads.
2373 void X86SpeculativeLoadHardeningPass::hardenReturnInstr(MachineInstr &MI) {
2374  MachineBasicBlock &MBB = *MI.getParent();
2375  DebugLoc Loc = MI.getDebugLoc();
2376  auto InsertPt = MI.getIterator();
2377 
2378  if (FenceCallAndRet)
2379  // No need to fence here as we'll fence at the return site itself. That
2380  // handles more cases than we can handle here.
2381  return;
2382 
2383  // Take our predicate state, shift it to the high 17 bits (so that we keep
2384  // pointers canonical) and merge it into RSP. This will allow the caller to
2385  // extract it when we return (speculatively).
2386  mergePredStateIntoSP(MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&MBB));
2387 }
2388 
2389 /// Trace the predicate state through a call.
2390 ///
2391 /// There are several layers of this needed to handle the full complexity of
2392 /// calls.
2393 ///
2394 /// First, we need to send the predicate state into the called function. We do
2395 /// this by merging it into the high bits of the stack pointer.
2396 ///
2397 /// For tail calls, this is all we need to do.
2398 ///
2399 /// For calls where we might return and resume the control flow, we need to
2400 /// extract the predicate state from the high bits of the stack pointer after
2401 /// control returns from the called function.
2402 ///
2403 /// We also need to verify that we intended to return to this location in the
2404 /// code. An attacker might arrange for the processor to mispredict the return
2405 /// to this valid but incorrect return address in the program rather than the
2406 /// correct one. See the paper on this attack, called "ret2spec" by the
2407 /// researchers, here:
2408 /// https://christian-rossow.de/publications/ret2spec-ccs2018.pdf
2409 ///
2410 /// The way we verify that we returned to the correct location is by preserving
2411 /// the expected return address across the call. One technique involves taking
2412 /// advantage of the red-zone to load the return address from `8(%rsp)` where it
2413 /// was left by the RET instruction when it popped `%rsp`. Alternatively, we can
2414 /// directly save the address into a register that will be preserved across the
2415 /// call. We compare this intended return address against the address
2416 /// immediately following the call (the observed return address). If these
2417 /// mismatch, we have detected misspeculation and can poison our predicate
2418 /// state.
2419 void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(
2420  MachineInstr &MI) {
2421  MachineBasicBlock &MBB = *MI.getParent();
2422  MachineFunction &MF = *MBB.getParent();
2423  auto InsertPt = MI.getIterator();
2424  DebugLoc Loc = MI.getDebugLoc();
2425 
2426  if (FenceCallAndRet) {
2427  if (MI.isReturn())
2428  // Tail call, we don't return to this function.
2429  // FIXME: We should also handle noreturn calls.
2430  return;
2431 
2432  // We don't need to fence before the call because the function should fence
2433  // in its entry. However, we do need to fence after the call returns.
2434  // Fencing before the return doesn't correctly handle cases where the return
2435  // itself is mispredicted.
2436  BuildMI(MBB, std::next(InsertPt), Loc, TII->get(X86::LFENCE));
2437  ++NumInstsInserted;
2438  ++NumLFENCEsInserted;
2439  return;
2440  }
2441 
2442  // First, we transfer the predicate state into the called function by merging
2443  // it into the stack pointer. This will kill the current def of the state.
2444  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2445  mergePredStateIntoSP(MBB, InsertPt, Loc, StateReg);
2446 
2447  // If this call is also a return, it is a tail call and we don't need anything
2448  // else to handle it so just return. Also, if there are no further
2449  // instructions and no successors, this call does not return so we can also
2450  // bail.
2451  if (MI.isReturn() || (std::next(InsertPt) == MBB.end() && MBB.succ_empty()))
2452  return;
2453 
2454  // Create a symbol to track the return address and attach it to the call
2455  // machine instruction. We will lower extra symbols attached to call
2456  // instructions as label immediately following the call.
2457  MCSymbol *RetSymbol =
2458  MF.getContext().createTempSymbol("slh_ret_addr",
2459  /*AlwaysAddSuffix*/ true);
2460  MI.setPostInstrSymbol(MF, RetSymbol);
2461 
2462  const TargetRegisterClass *AddrRC = &X86::GR64RegClass;
2463  unsigned ExpectedRetAddrReg = 0;
2464 
2465  // If we have no red zones or if the function returns twice (possibly without
2466  // using the `ret` instruction) like setjmp, we need to save the expected
2467  // return address prior to the call.
2468  if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone) ||
2469  MF.exposesReturnsTwice()) {
2470  // If we don't have red zones, we need to compute the expected return
2471  // address prior to the call and store it in a register that lives across
2472  // the call.
2473  //
2474  // In some ways, this is doubly satisfying as a mitigation because it will
2475  // also successfully detect stack smashing bugs in some cases (typically,
2476  // when a callee-saved register is used and the callee doesn't push it onto
2477  // the stack). But that isn't our primary goal, so we only use it as
2478  // a fallback.
2479  //
2480  // FIXME: It isn't clear that this is reliable in the face of
2481  // rematerialization in the register allocator. We somehow need to force
2482  // that to not occur for this particular instruction, and instead to spill
2483  // or otherwise preserve the value computed *prior* to the call.
2484  //
2485  // FIXME: It is even less clear why MachineCSE can't just fold this when we
2486  // end up having to use identical instructions both before and after the
2487  // call to feed the comparison.
2488  ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
2489  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
2490  !Subtarget->isPositionIndependent()) {
2491  BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64ri32), ExpectedRetAddrReg)
2492  .addSym(RetSymbol);
2493  } else {
2494  BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ExpectedRetAddrReg)
2495  .addReg(/*Base*/ X86::RIP)
2496  .addImm(/*Scale*/ 1)
2497  .addReg(/*Index*/ 0)
2498  .addSym(RetSymbol)
2499  .addReg(/*Segment*/ 0);
2500  }
2501  }
2502 
2503  // Step past the call to handle when it returns.
2504  ++InsertPt;
2505 
2506  // If we didn't pre-compute the expected return address into a register, then
2507  // red zones are enabled and the return address is still available on the
2508  // stack immediately after the call. As the very first instruction, we load it
2509  // into a register.
2510  if (!ExpectedRetAddrReg) {
2511  ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
2512  BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64rm), ExpectedRetAddrReg)
2513  .addReg(/*Base*/ X86::RSP)
2514  .addImm(/*Scale*/ 1)
2515  .addReg(/*Index*/ 0)
2516  .addImm(/*Displacement*/ -8) // The stack pointer has been popped, so
2517  // the return address is 8-bytes past it.
2518  .addReg(/*Segment*/ 0);
2519  }
2520 
2521  // Now we extract the callee's predicate state from the stack pointer.
2522  unsigned NewStateReg = extractPredStateFromSP(MBB, InsertPt, Loc);
2523 
2524  // Test the expected return address against our actual address. If we can
2525  // form this basic block's address as an immediate, this is easy. Otherwise
2526  // we compute it.
2527  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
2528  !Subtarget->isPositionIndependent()) {
2529  // FIXME: Could we fold this with the load? It would require careful EFLAGS
2530  // management.
2531  BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64ri32))
2532  .addReg(ExpectedRetAddrReg, RegState::Kill)
2533  .addSym(RetSymbol);
2534  } else {
2535  unsigned ActualRetAddrReg = MRI->createVirtualRegister(AddrRC);
2536  BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ActualRetAddrReg)
2537  .addReg(/*Base*/ X86::RIP)
2538  .addImm(/*Scale*/ 1)
2539  .addReg(/*Index*/ 0)
2540  .addSym(RetSymbol)
2541  .addReg(/*Segment*/ 0);
2542  BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64rr))
2543  .addReg(ExpectedRetAddrReg, RegState::Kill)
2544  .addReg(ActualRetAddrReg, RegState::Kill);
2545  }
2546 
2547  // Now conditionally update the predicate state we just extracted if we ended
2548  // up at a different return address than expected.
2549  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
2550  auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
2551 
2552  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
2553  auto CMovI = BuildMI(MBB, InsertPt, Loc, TII->get(CMovOp), UpdatedStateReg)
2554  .addReg(NewStateReg, RegState::Kill)
2555  .addReg(PS->PoisonReg)
2556  .addImm(X86::COND_NE);
2557  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
2558  ++NumInstsInserted;
2559  LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
2560 
2561  PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
2562 }
2563 
2564 /// An attacker may speculatively store over a value that is then speculatively
2565 /// loaded and used as the target of an indirect call or jump instruction. This
2566 /// is called Spectre v1.2 or Bounds Check Bypass Store (BCBS) and is described
2567 /// in this paper:
2568 /// https://people.csail.mit.edu/vlk/spectre11.pdf
2569 ///
2570 /// When this happens, the speculative execution of the call or jump will end up
2571 /// being steered to this attacker controlled address. While most such loads
2572 /// will be adequately hardened already, we want to ensure that they are
2573 /// definitively treated as needing post-load hardening. While address hardening
2574 /// is sufficient to prevent secret data from leaking to the attacker, it may
2575 /// not be sufficient to prevent an attacker from steering speculative
2576 /// execution. We forcibly unfolded all relevant loads above and so will always
2577 /// have an opportunity to post-load harden here, we just need to scan for cases
2578 /// not already flagged and add them.
2579 void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(
2580  MachineInstr &MI,
2581  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
2582  switch (MI.getOpcode()) {
2583  case X86::FARCALL16m:
2584  case X86::FARCALL32m:
2585  case X86::FARCALL64:
2586  case X86::FARJMP16m:
2587  case X86::FARJMP32m:
2588  case X86::FARJMP64:
2589  // We don't need to harden either far calls or far jumps as they are
2590  // safe from Spectre.
2591  return;
2592 
2593  default:
2594  break;
2595  }
2596 
2597  // We should never see a loading instruction at this point, as those should
2598  // have been unfolded.
2599  assert(!MI.mayLoad() && "Found a lingering loading instruction!");
2600 
2601  // If the first operand isn't a register, this is a branch or call
2602  // instruction with an immediate operand which doesn't need to be hardened.
2603  if (!MI.getOperand(0).isReg())
2604  return;
2605 
2606  // For all of these, the target register is the first operand of the
2607  // instruction.
2608  auto &TargetOp = MI.getOperand(0);
2609  unsigned OldTargetReg = TargetOp.getReg();
2610 
2611  // Try to lookup a hardened version of this register. We retain a reference
2612  // here as we want to update the map to track any newly computed hardened
2613  // register.
2614  unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];
2615 
2616  // If we don't have a hardened register yet, compute one. Otherwise, just use
2617  // the already hardened register.
2618  //
2619  // FIXME: It is a little suspect that we use partially hardened registers that
2620  // only feed addresses. The complexity of partial hardening with SHRX
2621  // continues to pile up. Should definitively measure its value and consider
2622  // eliminating it.
2623  if (!HardenedTargetReg)
2624  HardenedTargetReg = hardenValueInRegister(
2625  OldTargetReg, *MI.getParent(), MI.getIterator(), MI.getDebugLoc());
2626 
2627  // Set the target operand to the hardened register.
2628  TargetOp.setReg(HardenedTargetReg);
2629 
2630  ++NumCallsOrJumpsHardened;
2631 }
2632 
2633 INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY,
2634  "X86 speculative load hardener", false, false)
2635 INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass, PASS_KEY,
2636  "X86 speculative load hardener", false, false)
2637 
2639  return new X86SpeculativeLoadHardeningPass();
2640 }
MachineOperand * findRegisterDefOperand(unsigned Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
bool isCall(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:632
MachineBasicBlock * getMBB() const
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
bool addRegisterDead(unsigned Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound=false)
We have determined MI defined a register without a use.
void set(unsigned Idx)
void push_back(const T &Elt)
Definition: SmallVector.h:211
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:382
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
MachineOperand * findRegisterUseOperand(unsigned Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
unsigned Reg
static cl::opt< bool > HardenInterprocedurally(PASS_KEY "-ip", cl::desc("Harden interprocedurally by passing our state in and out of " "functions in the high bits of the stack pointer."), cl::init(true), cl::Hidden)
static const TargetRegisterClass * getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII, unsigned Opcode)
Compute the register class for the unfolded load.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:320
unsigned second
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1185
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
void setIsDead(bool Val=true)
static cl::opt< bool > FenceCallAndRet(PASS_KEY "-fence-call-and-ret", cl::desc("Use a full speculation fence to harden both call and ret edges " "rather than a lighter weight mitigation."), cl::init(false), cl::Hidden)
static bool hasVulnerableLoad(MachineFunction &MF)
Helper to scan a function for loads vulnerable to misspeculation that we want to harden.
void dump() const
dump - Print the current MachineFunction to cerr, useful for debugger use.
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
iterator_range< succ_iterator > successors()
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:343
bool test(unsigned Idx) const
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:648
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
static MachineBasicBlock & splitEdge(MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount, MachineInstr *Br, MachineInstr *&UncondBr, const X86InstrInfo &TII)
static cl::opt< bool > HardenEdgesWithLFENCE(PASS_KEY "-lfence", cl::desc("Use LFENCE along each conditional edge to harden against speculative " "loads rather than conditional movs and poisoned pointers."), cl::init(false), cl::Hidden)
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:266
static cl::opt< bool > EnablePostLoadHardening(PASS_KEY "-post-load", cl::desc("Harden the value loaded *after* it is loaded by " "flushing the loaded bits to 1. This is hard to do " "in general but can be done easily for GPRs."), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableSpeculativeLoadHardening("x86-speculative-load-hardening", cl::desc("Force enable speculative load hardening"), cl::init(false), cl::Hidden)
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
void clear()
Definition: SmallSet.h:218
static void canonicalizePHIOperands(MachineFunction &MF)
Removing duplicate PHI operands to leave the PHI in a canonical and predictable form.
static cl::opt< bool > HardenLoads(PASS_KEY "-loads", cl::desc("Sanitize loads from memory. When disable, no " "significant security is provided."), cl::init(true), cl::Hidden)
void dump() const
Definition: Pass.cpp:134
Memory SSA
Definition: MemorySSA.cpp:64
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol)
Set a symbol that will be emitted just after the instruction itself.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void Initialize(unsigned V)
Initialize - Reset this object to get ready for a new set of SSA updates.
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
Definition: MachineInstr.h:656
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false)
Return a cmov opcode for the given register size in bytes, and operand type.
bool isReturn(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:622
MCContext & getContext() const
#define PASS_KEY
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:427
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:220
unsigned const MachineRegisterInfo * MRI
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineInstrBuilder & UseMI
LLVM_NODISCARD bool empty() const
Definition: SmallPtrSet.h:91
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
void setMBB(MachineBasicBlock *MBB)
Represent the analysis usage information of a pass.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1192
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:381
unsigned getOperandBias(const MCInstrDesc &Desc)
getOperandBias - compute whether all of the def operands are repeated in the uses and therefore shoul...
Definition: X86BaseInfo.h:721
self_iterator getIterator()
Definition: ilist_node.h:81
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
iterator_range< pred_iterator > predecessors()
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setIsKill(bool Val=true)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1115
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
CondCode getCondFromBranch(const MachineInstr &MI)
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:297
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:377
void initializeX86SpeculativeLoadHardeningPassPass(PassRegistry &)
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
static bool isDataInvariant(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:374
const X86RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
Definition: X86InstrInfo.h:148
FunctionPass * createX86SpeculativeLoadHardeningPass()
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
const Function & getFunction() const
Return the LLVM function that this machine code represents.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2&#39;s erase_if which is equivalent t...
Definition: STLExtras.h:1374
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
CodeModel::Model getCodeModel() const
Returns the code model.
static cl::opt< bool > HardenIndirectCallsAndJumps(PASS_KEY "-indirect", cl::desc("Harden indirect calls and jumps against using speculatively " "stored attacker controlled addresses. This is designed to " "mitigate Spectre v1.2 style attacks."), cl::init(true), cl::Hidden)
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:63
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const TargetRegisterInfo &TRI)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
bool isEHPad() const
Returns true if the block is a landing pad.
bool verify(Pass *p=nullptr, const char *Banner=nullptr, bool AbortOnError=true) const
Run the current MachineFunction through the machine code verifier, useful for debugger use...
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
bool exposesReturnsTwice() const
exposesReturnsTwice - Returns true if the function calls setjmp or any other similar functions with a...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
unsigned GetValueInMiddleOfBlock(MachineBasicBlock *BB)
GetValueInMiddleOfBlock - Construct SSA form, materializing a value that is live in the middle of the...
void AddAvailableValue(MachineBasicBlock *BB, unsigned V)
AddAvailableValue - Indicate that a rewritten value is available at the end of the specified block wi...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:171
void splitSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New, bool NormalizeSuccProbs=false)
Split the old successor into old plus new and updates the probability info.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
X86 speculative load hardener
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition: MachineInstr.h:806
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY, "X86 speculative load hardener", false, false) INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass
static bool isDataInvariantLoad(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...
iterator SkipPHIsLabelsAndDebug(iterator I)
Return the first instruction in MBB after I that is not a PHI, label or debug.
unsigned getReg(unsigned Idx)
Get the register for the operand index.
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, unsigned *LoadRegIndex=nullptr) const override
getOpcodeAfterMemoryUnfold - Returns the opcode of the would be new instruction after load / store ar...
bool isImplicit() const
int getMemoryOperandNo(uint64_t TSFlags)
getMemoryOperandNo - The function returns the MCInst operand # for the first field of the memory oper...
Definition: X86BaseInfo.h:762
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164