LLVM  9.0.0svn
X86SpeculativeLoadHardening.cpp
Go to the documentation of this file.
1 //====- X86SpeculativeLoadHardening.cpp - A Spectre v1 mitigation ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// Provide a pass which mitigates speculative execution attacks which operate
11 /// by speculating incorrectly past some predicate (a type check, bounds check,
12 /// or other condition) to reach a load with invalid inputs and leak the data
13 /// accessed by that load using a side channel out of the speculative domain.
14 ///
15 /// For details on the attacks, see the first variant in both the Project Zero
16 /// writeup and the Spectre paper:
17 /// https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
18 /// https://spectreattack.com/spectre.pdf
19 ///
20 //===----------------------------------------------------------------------===//
21 
22 #include "X86.h"
23 #include "X86InstrBuilder.h"
24 #include "X86InstrInfo.h"
25 #include "X86Subtarget.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/DenseMap.h"
28 #include "llvm/ADT/Optional.h"
29 #include "llvm/ADT/STLExtras.h"
30 #include "llvm/ADT/ScopeExit.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallSet.h"
33 #include "llvm/ADT/SmallVector.h"
35 #include "llvm/ADT/Statistic.h"
50 #include "llvm/IR/DebugLoc.h"
51 #include "llvm/MC/MCSchedule.h"
52 #include "llvm/Pass.h"
54 #include "llvm/Support/Debug.h"
56 #include <algorithm>
57 #include <cassert>
58 #include <iterator>
59 #include <utility>
60 
61 using namespace llvm;
62 
63 #define PASS_KEY "x86-slh"
64 #define DEBUG_TYPE PASS_KEY
65 
66 STATISTIC(NumCondBranchesTraced, "Number of conditional branches traced");
67 STATISTIC(NumBranchesUntraced, "Number of branches unable to trace");
68 STATISTIC(NumAddrRegsHardened,
69  "Number of address mode used registers hardaned");
70 STATISTIC(NumPostLoadRegsHardened,
71  "Number of post-load register values hardened");
72 STATISTIC(NumCallsOrJumpsHardened,
73  "Number of calls or jumps requiring extra hardening");
74 STATISTIC(NumInstsInserted, "Number of instructions inserted");
75 STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
76 
78  "x86-speculative-load-hardening",
79  cl::desc("Force enable speculative load hardening"), cl::init(false),
80  cl::Hidden);
81 
83  PASS_KEY "-lfence",
84  cl::desc(
85  "Use LFENCE along each conditional edge to harden against speculative "
86  "loads rather than conditional movs and poisoned pointers."),
87  cl::init(false), cl::Hidden);
88 
90  PASS_KEY "-post-load",
91  cl::desc("Harden the value loaded *after* it is loaded by "
92  "flushing the loaded bits to 1. This is hard to do "
93  "in general but can be done easily for GPRs."),
94  cl::init(true), cl::Hidden);
95 
97  PASS_KEY "-fence-call-and-ret",
98  cl::desc("Use a full speculation fence to harden both call and ret edges "
99  "rather than a lighter weight mitigation."),
100  cl::init(false), cl::Hidden);
101 
103  PASS_KEY "-ip",
104  cl::desc("Harden interprocedurally by passing our state in and out of "
105  "functions in the high bits of the stack pointer."),
106  cl::init(true), cl::Hidden);
107 
108 static cl::opt<bool>
109  HardenLoads(PASS_KEY "-loads",
110  cl::desc("Sanitize loads from memory. When disable, no "
111  "significant security is provided."),
112  cl::init(true), cl::Hidden);
113 
115  PASS_KEY "-indirect",
116  cl::desc("Harden indirect calls and jumps against using speculatively "
117  "stored attacker controlled addresses. This is designed to "
118  "mitigate Spectre v1.2 style attacks."),
119  cl::init(true), cl::Hidden);
120 
121 namespace {
122 
123 class X86SpeculativeLoadHardeningPass : public MachineFunctionPass {
124 public:
125  X86SpeculativeLoadHardeningPass() : MachineFunctionPass(ID) {
128  }
129 
130  StringRef getPassName() const override {
131  return "X86 speculative load hardening";
132  }
133  bool runOnMachineFunction(MachineFunction &MF) override;
134  void getAnalysisUsage(AnalysisUsage &AU) const override;
135 
136  /// Pass identification, replacement for typeid.
137  static char ID;
138 
139 private:
140  /// The information about a block's conditional terminators needed to trace
141  /// our predicate state through the exiting edges.
142  struct BlockCondInfo {
143  MachineBasicBlock *MBB;
144 
145  // We mostly have one conditional branch, and in extremely rare cases have
146  // two. Three and more are so rare as to be unimportant for compile time.
148 
149  MachineInstr *UncondBr;
150  };
151 
152  /// Manages the predicate state traced through the program.
153  struct PredState {
154  unsigned InitialReg;
155  unsigned PoisonReg;
156 
157  const TargetRegisterClass *RC;
159 
160  PredState(MachineFunction &MF, const TargetRegisterClass *RC)
161  : RC(RC), SSA(MF) {}
162  };
163 
164  const X86Subtarget *Subtarget;
166  const X86InstrInfo *TII;
167  const TargetRegisterInfo *TRI;
168 
170 
171  void hardenEdgesWithLFENCE(MachineFunction &MF);
172 
173  SmallVector<BlockCondInfo, 16> collectBlockCondInfo(MachineFunction &MF);
174 
176  tracePredStateThroughCFG(MachineFunction &MF, ArrayRef<BlockCondInfo> Infos);
177 
178  void unfoldCallAndJumpLoads(MachineFunction &MF);
179 
181  tracePredStateThroughIndirectBranches(MachineFunction &MF);
182 
183  void tracePredStateThroughBlocksAndHarden(MachineFunction &MF);
184 
185  unsigned saveEFLAGS(MachineBasicBlock &MBB,
186  MachineBasicBlock::iterator InsertPt, DebugLoc Loc);
187  void restoreEFLAGS(MachineBasicBlock &MBB,
189  unsigned OFReg);
190 
191  void mergePredStateIntoSP(MachineBasicBlock &MBB,
193  unsigned PredStateReg);
194  unsigned extractPredStateFromSP(MachineBasicBlock &MBB,
196  DebugLoc Loc);
197 
198  void
199  hardenLoadAddr(MachineInstr &MI, MachineOperand &BaseMO,
200  MachineOperand &IndexMO,
201  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
202  MachineInstr *
203  sinkPostLoadHardenedInst(MachineInstr &MI,
204  SmallPtrSetImpl<MachineInstr *> &HardenedInstrs);
205  bool canHardenRegister(unsigned Reg);
206  unsigned hardenValueInRegister(unsigned Reg, MachineBasicBlock &MBB,
208  DebugLoc Loc);
209  unsigned hardenPostLoad(MachineInstr &MI);
210  void hardenReturnInstr(MachineInstr &MI);
211  void tracePredStateThroughCall(MachineInstr &MI);
212  void hardenIndirectCallOrJumpInstr(
213  MachineInstr &MI,
214  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
215 };
216 
217 } // end anonymous namespace
218 
220 
221 void X86SpeculativeLoadHardeningPass::getAnalysisUsage(
222  AnalysisUsage &AU) const {
224 }
225 
227  MachineBasicBlock &Succ, int SuccCount,
228  MachineInstr *Br, MachineInstr *&UncondBr,
229  const X86InstrInfo &TII) {
230  assert(!Succ.isEHPad() && "Shouldn't get edges to EH pads!");
231 
232  MachineFunction &MF = *MBB.getParent();
233 
234  MachineBasicBlock &NewMBB = *MF.CreateMachineBasicBlock();
235 
236  // We have to insert the new block immediately after the current one as we
237  // don't know what layout-successor relationships the successor has and we
238  // may not be able to (and generally don't want to) try to fix those up.
239  MF.insert(std::next(MachineFunction::iterator(&MBB)), &NewMBB);
240 
241  // Update the branch instruction if necessary.
242  if (Br) {
243  assert(Br->getOperand(0).getMBB() == &Succ &&
244  "Didn't start with the right target!");
245  Br->getOperand(0).setMBB(&NewMBB);
246 
247  // If this successor was reached through a branch rather than fallthrough,
248  // we might have *broken* fallthrough and so need to inject a new
249  // unconditional branch.
250  if (!UncondBr) {
251  MachineBasicBlock &OldLayoutSucc =
252  *std::next(MachineFunction::iterator(&NewMBB));
253  assert(MBB.isSuccessor(&OldLayoutSucc) &&
254  "Without an unconditional branch, the old layout successor should "
255  "be an actual successor!");
256  auto BrBuilder =
257  BuildMI(&MBB, DebugLoc(), TII.get(X86::JMP_1)).addMBB(&OldLayoutSucc);
258  // Update the unconditional branch now that we've added one.
259  UncondBr = &*BrBuilder;
260  }
261 
262  // Insert unconditional "jump Succ" instruction in the new block if
263  // necessary.
264  if (!NewMBB.isLayoutSuccessor(&Succ)) {
266  TII.insertBranch(NewMBB, &Succ, nullptr, Cond, Br->getDebugLoc());
267  }
268  } else {
269  assert(!UncondBr &&
270  "Cannot have a branchless successor and an unconditional branch!");
271  assert(NewMBB.isLayoutSuccessor(&Succ) &&
272  "A non-branch successor must have been a layout successor before "
273  "and now is a layout successor of the new block.");
274  }
275 
276  // If this is the only edge to the successor, we can just replace it in the
277  // CFG. Otherwise we need to add a new entry in the CFG for the new
278  // successor.
279  if (SuccCount == 1) {
280  MBB.replaceSuccessor(&Succ, &NewMBB);
281  } else {
282  MBB.splitSuccessor(&Succ, &NewMBB);
283  }
284 
285  // Hook up the edge from the new basic block to the old successor in the CFG.
286  NewMBB.addSuccessor(&Succ);
287 
288  // Fix PHI nodes in Succ so they refer to NewMBB instead of MBB.
289  for (MachineInstr &MI : Succ) {
290  if (!MI.isPHI())
291  break;
292  for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps;
293  OpIdx += 2) {
294  MachineOperand &OpV = MI.getOperand(OpIdx);
295  MachineOperand &OpMBB = MI.getOperand(OpIdx + 1);
296  assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!");
297  if (OpMBB.getMBB() != &MBB)
298  continue;
299 
300  // If this is the last edge to the succesor, just replace MBB in the PHI
301  if (SuccCount == 1) {
302  OpMBB.setMBB(&NewMBB);
303  break;
304  }
305 
306  // Otherwise, append a new pair of operands for the new incoming edge.
307  MI.addOperand(MF, OpV);
308  MI.addOperand(MF, MachineOperand::CreateMBB(&NewMBB));
309  break;
310  }
311  }
312 
313  // Inherit live-ins from the successor
314  for (auto &LI : Succ.liveins())
315  NewMBB.addLiveIn(LI);
316 
317  LLVM_DEBUG(dbgs() << " Split edge from '" << MBB.getName() << "' to '"
318  << Succ.getName() << "'.\n");
319  return NewMBB;
320 }
321 
322 /// Removing duplicate PHI operands to leave the PHI in a canonical and
323 /// predictable form.
324 ///
325 /// FIXME: It's really frustrating that we have to do this, but SSA-form in MIR
326 /// isn't what you might expect. We may have multiple entries in PHI nodes for
327 /// a single predecessor. This makes CFG-updating extremely complex, so here we
328 /// simplify all PHI nodes to a model even simpler than the IR's model: exactly
329 /// one entry per predecessor, regardless of how many edges there are.
332  SmallVector<int, 4> DupIndices;
333  for (auto &MBB : MF)
334  for (auto &MI : MBB) {
335  if (!MI.isPHI())
336  break;
337 
338  // First we scan the operands of the PHI looking for duplicate entries
339  // a particular predecessor. We retain the operand index of each duplicate
340  // entry found.
341  for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps;
342  OpIdx += 2)
343  if (!Preds.insert(MI.getOperand(OpIdx + 1).getMBB()).second)
344  DupIndices.push_back(OpIdx);
345 
346  // Now walk the duplicate indices, removing both the block and value. Note
347  // that these are stored as a vector making this element-wise removal
348  // :w
349  // potentially quadratic.
350  //
351  // FIXME: It is really frustrating that we have to use a quadratic
352  // removal algorithm here. There should be a better way, but the use-def
353  // updates required make that impossible using the public API.
354  //
355  // Note that we have to process these backwards so that we don't
356  // invalidate other indices with each removal.
357  while (!DupIndices.empty()) {
358  int OpIdx = DupIndices.pop_back_val();
359  // Remove both the block and value operand, again in reverse order to
360  // preserve indices.
361  MI.RemoveOperand(OpIdx + 1);
362  MI.RemoveOperand(OpIdx);
363  }
364 
365  Preds.clear();
366  }
367 }
368 
369 /// Helper to scan a function for loads vulnerable to misspeculation that we
370 /// want to harden.
371 ///
372 /// We use this to avoid making changes to functions where there is nothing we
373 /// need to do to harden against misspeculation.
375  for (MachineBasicBlock &MBB : MF) {
376  for (MachineInstr &MI : MBB) {
377  // Loads within this basic block after an LFENCE are not at risk of
378  // speculatively executing with invalid predicates from prior control
379  // flow. So break out of this block but continue scanning the function.
380  if (MI.getOpcode() == X86::LFENCE)
381  break;
382 
383  // Looking for loads only.
384  if (!MI.mayLoad())
385  continue;
386 
387  // An MFENCE is modeled as a load but isn't vulnerable to misspeculation.
388  if (MI.getOpcode() == X86::MFENCE)
389  continue;
390 
391  // We found a load.
392  return true;
393  }
394  }
395 
396  // No loads found.
397  return false;
398 }
399 
400 bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
401  MachineFunction &MF) {
402  LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
403  << " **********\n");
404 
405  // Only run if this pass is forced enabled or we detect the relevant function
406  // attribute requesting SLH.
408  !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
409  return false;
410 
411  Subtarget = &MF.getSubtarget<X86Subtarget>();
412  MRI = &MF.getRegInfo();
413  TII = Subtarget->getInstrInfo();
414  TRI = Subtarget->getRegisterInfo();
415 
416  // FIXME: Support for 32-bit.
417  PS.emplace(MF, &X86::GR64_NOSPRegClass);
418 
419  if (MF.begin() == MF.end())
420  // Nothing to do for a degenerate empty function...
421  return false;
422 
423  // We support an alternative hardening technique based on a debug flag.
424  if (HardenEdgesWithLFENCE) {
425  hardenEdgesWithLFENCE(MF);
426  return true;
427  }
428 
429  // Create a dummy debug loc to use for all the generated code here.
430  DebugLoc Loc;
431 
432  MachineBasicBlock &Entry = *MF.begin();
433  auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());
434 
435  // Do a quick scan to see if we have any checkable loads.
436  bool HasVulnerableLoad = hasVulnerableLoad(MF);
437 
438  // See if we have any conditional branching blocks that we will need to trace
439  // predicate state through.
440  SmallVector<BlockCondInfo, 16> Infos = collectBlockCondInfo(MF);
441 
442  // If we have no interesting conditions or loads, nothing to do here.
443  if (!HasVulnerableLoad && Infos.empty())
444  return true;
445 
446  // The poison value is required to be an all-ones value for many aspects of
447  // this mitigation.
448  const int PoisonVal = -1;
449  PS->PoisonReg = MRI->createVirtualRegister(PS->RC);
450  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64ri32), PS->PoisonReg)
451  .addImm(PoisonVal);
452  ++NumInstsInserted;
453 
454  // If we have loads being hardened and we've asked for call and ret edges to
455  // get a full fence-based mitigation, inject that fence.
456  if (HasVulnerableLoad && FenceCallAndRet) {
457  // We need to insert an LFENCE at the start of the function to suspend any
458  // incoming misspeculation from the caller. This helps two-fold: the caller
459  // may not have been protected as this code has been, and this code gets to
460  // not take any specific action to protect across calls.
461  // FIXME: We could skip this for functions which unconditionally return
462  // a constant.
463  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::LFENCE));
464  ++NumInstsInserted;
465  ++NumLFENCEsInserted;
466  }
467 
468  // If we guarded the entry with an LFENCE and have no conditionals to protect
469  // in blocks, then we're done.
470  if (FenceCallAndRet && Infos.empty())
471  // We may have changed the function's code at this point to insert fences.
472  return true;
473 
474  // For every basic block in the function which can b
476  // Set up the predicate state by extracting it from the incoming stack
477  // pointer so we pick up any misspeculation in our caller.
478  PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
479  } else {
480  // Otherwise, just build the predicate state itself by zeroing a register
481  // as we don't need any initial state.
482  PS->InitialReg = MRI->createVirtualRegister(PS->RC);
483  unsigned PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass);
484  auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0),
485  PredStateSubReg);
486  ++NumInstsInserted;
487  MachineOperand *ZeroEFLAGSDefOp =
488  ZeroI->findRegisterDefOperand(X86::EFLAGS);
489  assert(ZeroEFLAGSDefOp && ZeroEFLAGSDefOp->isImplicit() &&
490  "Must have an implicit def of EFLAGS!");
491  ZeroEFLAGSDefOp->setIsDead(true);
492  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG),
493  PS->InitialReg)
494  .addImm(0)
495  .addReg(PredStateSubReg)
496  .addImm(X86::sub_32bit);
497  }
498 
499  // We're going to need to trace predicate state throughout the function's
500  // CFG. Prepare for this by setting up our initial state of PHIs with unique
501  // predecessor entries and all the initial predicate state.
503 
504  // Track the updated values in an SSA updater to rewrite into SSA form at the
505  // end.
506  PS->SSA.Initialize(PS->InitialReg);
507  PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);
508 
509  // Trace through the CFG.
510  auto CMovs = tracePredStateThroughCFG(MF, Infos);
511 
512  // We may also enter basic blocks in this function via exception handling
513  // control flow. Here, if we are hardening interprocedurally, we need to
514  // re-capture the predicate state from the throwing code. In the Itanium ABI,
515  // the throw will always look like a call to __cxa_throw and will have the
516  // predicate state in the stack pointer, so extract fresh predicate state from
517  // the stack pointer and make it available in SSA.
518  // FIXME: Handle non-itanium ABI EH models.
520  for (MachineBasicBlock &MBB : MF) {
521  assert(!MBB.isEHScopeEntry() && "Only Itanium ABI EH supported!");
522  assert(!MBB.isEHFuncletEntry() && "Only Itanium ABI EH supported!");
523  assert(!MBB.isCleanupFuncletEntry() && "Only Itanium ABI EH supported!");
524  if (!MBB.isEHPad())
525  continue;
526  PS->SSA.AddAvailableValue(
527  &MBB,
528  extractPredStateFromSP(MBB, MBB.SkipPHIsAndLabels(MBB.begin()), Loc));
529  }
530  }
531 
533  // If we are going to harden calls and jumps we need to unfold their memory
534  // operands.
535  unfoldCallAndJumpLoads(MF);
536 
537  // Then we trace predicate state through the indirect branches.
538  auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);
539  CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());
540  }
541 
542  // Now that we have the predicate state available at the start of each block
543  // in the CFG, trace it through each block, hardening vulnerable instructions
544  // as we go.
545  tracePredStateThroughBlocksAndHarden(MF);
546 
547  // Now rewrite all the uses of the pred state using the SSA updater to insert
548  // PHIs connecting the state between blocks along the CFG edges.
549  for (MachineInstr *CMovI : CMovs)
550  for (MachineOperand &Op : CMovI->operands()) {
551  if (!Op.isReg() || Op.getReg() != PS->InitialReg)
552  continue;
553 
554  PS->SSA.RewriteUse(Op);
555  }
556 
557  LLVM_DEBUG(dbgs() << "Final speculative load hardened function:\n"; MF.dump();
558  dbgs() << "\n"; MF.verify(this));
559  return true;
560 }
561 
562 /// Implements the naive hardening approach of putting an LFENCE after every
563 /// potentially mis-predicted control flow construct.
564 ///
565 /// We include this as an alternative mostly for the purpose of comparison. The
566 /// performance impact of this is expected to be extremely severe and not
567 /// practical for any real-world users.
568 void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(
569  MachineFunction &MF) {
570  // First, we scan the function looking for blocks that are reached along edges
571  // that we might want to harden.
573  for (MachineBasicBlock &MBB : MF) {
574  // If there are no or only one successor, nothing to do here.
575  if (MBB.succ_size() <= 1)
576  continue;
577 
578  // Skip blocks unless their terminators start with a branch. Other
579  // terminators don't seem interesting for guarding against misspeculation.
580  auto TermIt = MBB.getFirstTerminator();
581  if (TermIt == MBB.end() || !TermIt->isBranch())
582  continue;
583 
584  // Add all the non-EH-pad succossors to the blocks we want to harden. We
585  // skip EH pads because there isn't really a condition of interest on
586  // entering.
587  for (MachineBasicBlock *SuccMBB : MBB.successors())
588  if (!SuccMBB->isEHPad())
589  Blocks.insert(SuccMBB);
590  }
591 
592  for (MachineBasicBlock *MBB : Blocks) {
593  auto InsertPt = MBB->SkipPHIsAndLabels(MBB->begin());
594  BuildMI(*MBB, InsertPt, DebugLoc(), TII->get(X86::LFENCE));
595  ++NumInstsInserted;
596  ++NumLFENCEsInserted;
597  }
598 }
599 
601 X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) {
603 
604  // Walk the function and build up a summary for each block's conditions that
605  // we need to trace through.
606  for (MachineBasicBlock &MBB : MF) {
607  // If there are no or only one successor, nothing to do here.
608  if (MBB.succ_size() <= 1)
609  continue;
610 
611  // We want to reliably handle any conditional branch terminators in the
612  // MBB, so we manually analyze the branch. We can handle all of the
613  // permutations here, including ones that analyze branch cannot.
614  //
615  // The approach is to walk backwards across the terminators, resetting at
616  // any unconditional non-indirect branch, and track all conditional edges
617  // to basic blocks as well as the fallthrough or unconditional successor
618  // edge. For each conditional edge, we track the target and the opposite
619  // condition code in order to inject a "no-op" cmov into that successor
620  // that will harden the predicate. For the fallthrough/unconditional
621  // edge, we inject a separate cmov for each conditional branch with
622  // matching condition codes. This effectively implements an "and" of the
623  // condition flags, even if there isn't a single condition flag that would
624  // directly implement that. We don't bother trying to optimize either of
625  // these cases because if such an optimization is possible, LLVM should
626  // have optimized the conditional *branches* in that way already to reduce
627  // instruction count. This late, we simply assume the minimal number of
628  // branch instructions is being emitted and use that to guide our cmov
629  // insertion.
630 
631  BlockCondInfo Info = {&MBB, {}, nullptr};
632 
633  // Now walk backwards through the terminators and build up successors they
634  // reach and the conditions.
635  for (MachineInstr &MI : llvm::reverse(MBB)) {
636  // Once we've handled all the terminators, we're done.
637  if (!MI.isTerminator())
638  break;
639 
640  // If we see a non-branch terminator, we can't handle anything so bail.
641  if (!MI.isBranch()) {
642  Info.CondBrs.clear();
643  break;
644  }
645 
646  // If we see an unconditional branch, reset our state, clear any
647  // fallthrough, and set this is the "else" successor.
648  if (MI.getOpcode() == X86::JMP_1) {
649  Info.CondBrs.clear();
650  Info.UncondBr = &MI;
651  continue;
652  }
653 
654  // If we get an invalid condition, we have an indirect branch or some
655  // other unanalyzable "fallthrough" case. We model this as a nullptr for
656  // the destination so we can still guard any conditional successors.
657  // Consider code sequences like:
658  // ```
659  // jCC L1
660  // jmpq *%rax
661  // ```
662  // We still want to harden the edge to `L1`.
663  if (X86::getCondFromBranchOpc(MI.getOpcode()) == X86::COND_INVALID) {
664  Info.CondBrs.clear();
665  Info.UncondBr = &MI;
666  continue;
667  }
668 
669  // We have a vanilla conditional branch, add it to our list.
670  Info.CondBrs.push_back(&MI);
671  }
672  if (Info.CondBrs.empty()) {
673  ++NumBranchesUntraced;
674  LLVM_DEBUG(dbgs() << "WARNING: unable to secure successors of block:\n";
675  MBB.dump());
676  continue;
677  }
678 
679  Infos.push_back(Info);
680  }
681 
682  return Infos;
683 }
684 
685 /// Trace the predicate state through the CFG, instrumenting each conditional
686 /// branch such that misspeculation through an edge will poison the predicate
687 /// state.
688 ///
689 /// Returns the list of inserted CMov instructions so that they can have their
690 /// uses of the predicate state rewritten into proper SSA form once it is
691 /// complete.
693 X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(
695  // Collect the inserted cmov instructions so we can rewrite their uses of the
696  // predicate state into SSA form.
698 
699  // Now walk all of the basic blocks looking for ones that end in conditional
700  // jumps where we need to update this register along each edge.
701  for (const BlockCondInfo &Info : Infos) {
702  MachineBasicBlock &MBB = *Info.MBB;
703  const SmallVectorImpl<MachineInstr *> &CondBrs = Info.CondBrs;
704  MachineInstr *UncondBr = Info.UncondBr;
705 
706  LLVM_DEBUG(dbgs() << "Tracing predicate through block: " << MBB.getName()
707  << "\n");
708  ++NumCondBranchesTraced;
709 
710  // Compute the non-conditional successor as either the target of any
711  // unconditional branch or the layout successor.
712  MachineBasicBlock *UncondSucc =
713  UncondBr ? (UncondBr->getOpcode() == X86::JMP_1
714  ? UncondBr->getOperand(0).getMBB()
715  : nullptr)
716  : &*std::next(MachineFunction::iterator(&MBB));
717 
718  // Count how many edges there are to any given successor.
720  if (UncondSucc)
721  ++SuccCounts[UncondSucc];
722  for (auto *CondBr : CondBrs)
723  ++SuccCounts[CondBr->getOperand(0).getMBB()];
724 
725  // A lambda to insert cmov instructions into a block checking all of the
726  // condition codes in a sequence.
727  auto BuildCheckingBlockForSuccAndConds =
728  [&](MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount,
729  MachineInstr *Br, MachineInstr *&UncondBr,
730  ArrayRef<X86::CondCode> Conds) {
731  // First, we split the edge to insert the checking block into a safe
732  // location.
733  auto &CheckingMBB =
734  (SuccCount == 1 && Succ.pred_size() == 1)
735  ? Succ
736  : splitEdge(MBB, Succ, SuccCount, Br, UncondBr, *TII);
737 
738  bool LiveEFLAGS = Succ.isLiveIn(X86::EFLAGS);
739  if (!LiveEFLAGS)
740  CheckingMBB.addLiveIn(X86::EFLAGS);
741 
742  // Now insert the cmovs to implement the checks.
743  auto InsertPt = CheckingMBB.begin();
744  assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&
745  "Should never have a PHI in the initial checking block as it "
746  "always has a single predecessor!");
747 
748  // We will wire each cmov to each other, but need to start with the
749  // incoming pred state.
750  unsigned CurStateReg = PS->InitialReg;
751 
752  for (X86::CondCode Cond : Conds) {
753  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
754  auto CMovOp = X86::getCMovFromCond(Cond, PredStateSizeInBytes);
755 
756  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
757  // Note that we intentionally use an empty debug location so that
758  // this picks up the preceding location.
759  auto CMovI = BuildMI(CheckingMBB, InsertPt, DebugLoc(),
760  TII->get(CMovOp), UpdatedStateReg)
761  .addReg(CurStateReg)
762  .addReg(PS->PoisonReg);
763  // If this is the last cmov and the EFLAGS weren't originally
764  // live-in, mark them as killed.
765  if (!LiveEFLAGS && Cond == Conds.back())
766  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
767 
768  ++NumInstsInserted;
769  LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump();
770  dbgs() << "\n");
771 
772  // The first one of the cmovs will be using the top level
773  // `PredStateReg` and need to get rewritten into SSA form.
774  if (CurStateReg == PS->InitialReg)
775  CMovs.push_back(&*CMovI);
776 
777  // The next cmov should start from this one's def.
778  CurStateReg = UpdatedStateReg;
779  }
780 
781  // And put the last one into the available values for SSA form of our
782  // predicate state.
783  PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);
784  };
785 
786  std::vector<X86::CondCode> UncondCodeSeq;
787  for (auto *CondBr : CondBrs) {
788  MachineBasicBlock &Succ = *CondBr->getOperand(0).getMBB();
789  int &SuccCount = SuccCounts[&Succ];
790 
791  X86::CondCode Cond = X86::getCondFromBranchOpc(CondBr->getOpcode());
793  UncondCodeSeq.push_back(Cond);
794 
795  BuildCheckingBlockForSuccAndConds(MBB, Succ, SuccCount, CondBr, UncondBr,
796  {InvCond});
797 
798  // Decrement the successor count now that we've split one of the edges.
799  // We need to keep the count of edges to the successor accurate in order
800  // to know above when to *replace* the successor in the CFG vs. just
801  // adding the new successor.
802  --SuccCount;
803  }
804 
805  // Since we may have split edges and changed the number of successors,
806  // normalize the probabilities. This avoids doing it each time we split an
807  // edge.
808  MBB.normalizeSuccProbs();
809 
810  // Finally, we need to insert cmovs into the "fallthrough" edge. Here, we
811  // need to intersect the other condition codes. We can do this by just
812  // doing a cmov for each one.
813  if (!UncondSucc)
814  // If we have no fallthrough to protect (perhaps it is an indirect jump?)
815  // just skip this and continue.
816  continue;
817 
818  assert(SuccCounts[UncondSucc] == 1 &&
819  "We should never have more than one edge to the unconditional "
820  "successor at this point because every other edge must have been "
821  "split above!");
822 
823  // Sort and unique the codes to minimize them.
824  llvm::sort(UncondCodeSeq);
825  UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()),
826  UncondCodeSeq.end());
827 
828  // Build a checking version of the successor.
829  BuildCheckingBlockForSuccAndConds(MBB, *UncondSucc, /*SuccCount*/ 1,
830  UncondBr, UncondBr, UncondCodeSeq);
831  }
832 
833  return CMovs;
834 }
835 
836 /// Compute the register class for the unfolded load.
837 ///
838 /// FIXME: This should probably live in X86InstrInfo, potentially by adding
839 /// a way to unfold into a newly created vreg rather than requiring a register
840 /// input.
841 static const TargetRegisterClass *
843  unsigned Opcode) {
844  unsigned Index;
845  unsigned UnfoldedOpc = TII.getOpcodeAfterMemoryUnfold(
846  Opcode, /*UnfoldLoad*/ true, /*UnfoldStore*/ false, &Index);
847  const MCInstrDesc &MCID = TII.get(UnfoldedOpc);
848  return TII.getRegClass(MCID, Index, &TII.getRegisterInfo(), MF);
849 }
850 
851 void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
852  MachineFunction &MF) {
853  for (MachineBasicBlock &MBB : MF)
854  for (auto MII = MBB.instr_begin(), MIE = MBB.instr_end(); MII != MIE;) {
855  // Grab a reference and increment the iterator so we can remove this
856  // instruction if needed without disturbing the iteration.
857  MachineInstr &MI = *MII++;
858 
859  // Must either be a call or a branch.
860  if (!MI.isCall() && !MI.isBranch())
861  continue;
862  // We only care about loading variants of these instructions.
863  if (!MI.mayLoad())
864  continue;
865 
866  switch (MI.getOpcode()) {
867  default: {
868  LLVM_DEBUG(
869  dbgs() << "ERROR: Found an unexpected loading branch or call "
870  "instruction:\n";
871  MI.dump(); dbgs() << "\n");
872  report_fatal_error("Unexpected loading branch or call!");
873  }
874 
875  case X86::FARCALL16m:
876  case X86::FARCALL32m:
877  case X86::FARCALL64:
878  case X86::FARJMP16m:
879  case X86::FARJMP32m:
880  case X86::FARJMP64:
881  // We cannot mitigate far jumps or calls, but we also don't expect them
882  // to be vulnerable to Spectre v1.2 style attacks.
883  continue;
884 
885  case X86::CALL16m:
886  case X86::CALL16m_NT:
887  case X86::CALL32m:
888  case X86::CALL32m_NT:
889  case X86::CALL64m:
890  case X86::CALL64m_NT:
891  case X86::JMP16m:
892  case X86::JMP16m_NT:
893  case X86::JMP32m:
894  case X86::JMP32m_NT:
895  case X86::JMP64m:
896  case X86::JMP64m_NT:
897  case X86::TAILJMPm64:
898  case X86::TAILJMPm64_REX:
899  case X86::TAILJMPm:
900  case X86::TCRETURNmi64:
901  case X86::TCRETURNmi: {
902  // Use the generic unfold logic now that we know we're dealing with
903  // expected instructions.
904  // FIXME: We don't have test coverage for all of these!
905  auto *UnfoldedRC = getRegClassForUnfoldedLoad(MF, *TII, MI.getOpcode());
906  if (!UnfoldedRC) {
907  LLVM_DEBUG(dbgs()
908  << "ERROR: Unable to unfold load from instruction:\n";
909  MI.dump(); dbgs() << "\n");
910  report_fatal_error("Unable to unfold load!");
911  }
912  unsigned Reg = MRI->createVirtualRegister(UnfoldedRC);
914  // If we were able to compute an unfolded reg class, any failure here
915  // is just a programming error so just assert.
916  bool Unfolded =
917  TII->unfoldMemoryOperand(MF, MI, Reg, /*UnfoldLoad*/ true,
918  /*UnfoldStore*/ false, NewMIs);
919  (void)Unfolded;
920  assert(Unfolded &&
921  "Computed unfolded register class but failed to unfold");
922  // Now stitch the new instructions into place and erase the old one.
923  for (auto *NewMI : NewMIs)
924  MBB.insert(MI.getIterator(), NewMI);
925  MI.eraseFromParent();
926  LLVM_DEBUG({
927  dbgs() << "Unfolded load successfully into:\n";
928  for (auto *NewMI : NewMIs) {
929  NewMI->dump();
930  dbgs() << "\n";
931  }
932  });
933  continue;
934  }
935  }
936  llvm_unreachable("Escaped switch with default!");
937  }
938 }
939 
940 /// Trace the predicate state through indirect branches, instrumenting them to
941 /// poison the state if a target is reached that does not match the expected
942 /// target.
943 ///
944 /// This is designed to mitigate Spectre variant 1 attacks where an indirect
945 /// branch is trained to predict a particular target and then mispredicts that
946 /// target in a way that can leak data. Despite using an indirect branch, this
947 /// is really a variant 1 style attack: it does not steer execution to an
948 /// arbitrary or attacker controlled address, and it does not require any
949 /// special code executing next to the victim. This attack can also be mitigated
950 /// through retpolines, but those require either replacing indirect branches
951 /// with conditional direct branches or lowering them through a device that
952 /// blocks speculation. This mitigation can replace these retpoline-style
953 /// mitigations for jump tables and other indirect branches within a function
954 /// when variant 2 isn't a risk while allowing limited speculation. Indirect
955 /// calls, however, cannot be mitigated through this technique without changing
956 /// the ABI in a fundamental way.
958 X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(
959  MachineFunction &MF) {
960  // We use the SSAUpdater to insert PHI nodes for the target addresses of
961  // indirect branches. We don't actually need the full power of the SSA updater
962  // in this particular case as we always have immediately available values, but
963  // this avoids us having to re-implement the PHI construction logic.
964  MachineSSAUpdater TargetAddrSSA(MF);
965  TargetAddrSSA.Initialize(MRI->createVirtualRegister(&X86::GR64RegClass));
966 
967  // Track which blocks were terminated with an indirect branch.
968  SmallPtrSet<MachineBasicBlock *, 4> IndirectTerminatedMBBs;
969 
970  // We need to know what blocks end up reached via indirect branches. We
971  // expect this to be a subset of those whose address is taken and so track it
972  // directly via the CFG.
973  SmallPtrSet<MachineBasicBlock *, 4> IndirectTargetMBBs;
974 
975  // Walk all the blocks which end in an indirect branch and make the
976  // target address available.
977  for (MachineBasicBlock &MBB : MF) {
978  // Find the last terminator.
979  auto MII = MBB.instr_rbegin();
980  while (MII != MBB.instr_rend() && MII->isDebugInstr())
981  ++MII;
982  if (MII == MBB.instr_rend())
983  continue;
984  MachineInstr &TI = *MII;
985  if (!TI.isTerminator() || !TI.isBranch())
986  // No terminator or non-branch terminator.
987  continue;
988 
989  unsigned TargetReg;
990 
991  switch (TI.getOpcode()) {
992  default:
993  // Direct branch or conditional branch (leading to fallthrough).
994  continue;
995 
996  case X86::FARJMP16m:
997  case X86::FARJMP32m:
998  case X86::FARJMP64:
999  // We cannot mitigate far jumps or calls, but we also don't expect them
1000  // to be vulnerable to Spectre v1.2 or v2 (self trained) style attacks.
1001  continue;
1002 
1003  case X86::JMP16m:
1004  case X86::JMP16m_NT:
1005  case X86::JMP32m:
1006  case X86::JMP32m_NT:
1007  case X86::JMP64m:
1008  case X86::JMP64m_NT:
1009  // Mostly as documentation.
1010  report_fatal_error("Memory operand jumps should have been unfolded!");
1011 
1012  case X86::JMP16r:
1014  "Support for 16-bit indirect branches is not implemented.");
1015  case X86::JMP32r:
1017  "Support for 32-bit indirect branches is not implemented.");
1018 
1019  case X86::JMP64r:
1020  TargetReg = TI.getOperand(0).getReg();
1021  }
1022 
1023  // We have definitely found an indirect branch. Verify that there are no
1024  // preceding conditional branches as we don't yet support that.
1025  if (llvm::any_of(MBB.terminators(), [&](MachineInstr &OtherTI) {
1026  return !OtherTI.isDebugInstr() && &OtherTI != &TI;
1027  })) {
1028  LLVM_DEBUG({
1029  dbgs() << "ERROR: Found other terminators in a block with an indirect "
1030  "branch! This is not yet supported! Terminator sequence:\n";
1031  for (MachineInstr &MI : MBB.terminators()) {
1032  MI.dump();
1033  dbgs() << '\n';
1034  }
1035  });
1036  report_fatal_error("Unimplemented terminator sequence!");
1037  }
1038 
1039  // Make the target register an available value for this block.
1040  TargetAddrSSA.AddAvailableValue(&MBB, TargetReg);
1041  IndirectTerminatedMBBs.insert(&MBB);
1042 
1043  // Add all the successors to our target candidates.
1044  for (MachineBasicBlock *Succ : MBB.successors())
1045  IndirectTargetMBBs.insert(Succ);
1046  }
1047 
1048  // Keep track of the cmov instructions we insert so we can return them.
1050 
1051  // If we didn't find any indirect branches with targets, nothing to do here.
1052  if (IndirectTargetMBBs.empty())
1053  return CMovs;
1054 
1055  // We found indirect branches and targets that need to be instrumented to
1056  // harden loads within them. Walk the blocks of the function (to get a stable
1057  // ordering) and instrument each target of an indirect branch.
1058  for (MachineBasicBlock &MBB : MF) {
1059  // Skip the blocks that aren't candidate targets.
1060  if (!IndirectTargetMBBs.count(&MBB))
1061  continue;
1062 
1063  // We don't expect EH pads to ever be reached via an indirect branch. If
1064  // this is desired for some reason, we could simply skip them here rather
1065  // than asserting.
1066  assert(!MBB.isEHPad() &&
1067  "Unexpected EH pad as target of an indirect branch!");
1068 
1069  // We should never end up threading EFLAGS into a block to harden
1070  // conditional jumps as there would be an additional successor via the
1071  // indirect branch. As a consequence, all such edges would be split before
1072  // reaching here, and the inserted block will handle the EFLAGS-based
1073  // hardening.
1074  assert(!MBB.isLiveIn(X86::EFLAGS) &&
1075  "Cannot check within a block that already has live-in EFLAGS!");
1076 
1077  // We can't handle having non-indirect edges into this block unless this is
1078  // the only successor and we can synthesize the necessary target address.
1079  for (MachineBasicBlock *Pred : MBB.predecessors()) {
1080  // If we've already handled this by extracting the target directly,
1081  // nothing to do.
1082  if (IndirectTerminatedMBBs.count(Pred))
1083  continue;
1084 
1085  // Otherwise, we have to be the only successor. We generally expect this
1086  // to be true as conditional branches should have had a critical edge
1087  // split already. We don't however need to worry about EH pad successors
1088  // as they'll happily ignore the target and their hardening strategy is
1089  // resilient to all ways in which they could be reached speculatively.
1090  if (!llvm::all_of(Pred->successors(), [&](MachineBasicBlock *Succ) {
1091  return Succ->isEHPad() || Succ == &MBB;
1092  })) {
1093  LLVM_DEBUG({
1094  dbgs() << "ERROR: Found conditional entry to target of indirect "
1095  "branch!\n";
1096  Pred->dump();
1097  MBB.dump();
1098  });
1099  report_fatal_error("Cannot harden a conditional entry to a target of "
1100  "an indirect branch!");
1101  }
1102 
1103  // Now we need to compute the address of this block and install it as a
1104  // synthetic target in the predecessor. We do this at the bottom of the
1105  // predecessor.
1106  auto InsertPt = Pred->getFirstTerminator();
1107  unsigned TargetReg = MRI->createVirtualRegister(&X86::GR64RegClass);
1108  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
1109  !Subtarget->isPositionIndependent()) {
1110  // Directly materialize it into an immediate.
1111  auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(),
1112  TII->get(X86::MOV64ri32), TargetReg)
1113  .addMBB(&MBB);
1114  ++NumInstsInserted;
1115  (void)AddrI;
1116  LLVM_DEBUG(dbgs() << " Inserting mov: "; AddrI->dump();
1117  dbgs() << "\n");
1118  } else {
1119  auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(), TII->get(X86::LEA64r),
1120  TargetReg)
1121  .addReg(/*Base*/ X86::RIP)
1122  .addImm(/*Scale*/ 1)
1123  .addReg(/*Index*/ 0)
1124  .addMBB(&MBB)
1125  .addReg(/*Segment*/ 0);
1126  ++NumInstsInserted;
1127  (void)AddrI;
1128  LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump();
1129  dbgs() << "\n");
1130  }
1131  // And make this available.
1132  TargetAddrSSA.AddAvailableValue(Pred, TargetReg);
1133  }
1134 
1135  // Materialize the needed SSA value of the target. Note that we need the
1136  // middle of the block as this block might at the bottom have an indirect
1137  // branch back to itself. We can do this here because at this point, every
1138  // predecessor of this block has an available value. This is basically just
1139  // automating the construction of a PHI node for this target.
1140  unsigned TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&MBB);
1141 
1142  // Insert a comparison of the incoming target register with this block's
1143  // address. This also requires us to mark the block as having its address
1144  // taken explicitly.
1145  MBB.setHasAddressTaken();
1146  auto InsertPt = MBB.SkipPHIsLabelsAndDebug(MBB.begin());
1147  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
1148  !Subtarget->isPositionIndependent()) {
1149  // Check directly against a relocated immediate when we can.
1150  auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64ri32))
1151  .addReg(TargetReg, RegState::Kill)
1152  .addMBB(&MBB);
1153  ++NumInstsInserted;
1154  (void)CheckI;
1155  LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
1156  } else {
1157  // Otherwise compute the address into a register first.
1158  unsigned AddrReg = MRI->createVirtualRegister(&X86::GR64RegClass);
1159  auto AddrI =
1160  BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::LEA64r), AddrReg)
1161  .addReg(/*Base*/ X86::RIP)
1162  .addImm(/*Scale*/ 1)
1163  .addReg(/*Index*/ 0)
1164  .addMBB(&MBB)
1165  .addReg(/*Segment*/ 0);
1166  ++NumInstsInserted;
1167  (void)AddrI;
1168  LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump(); dbgs() << "\n");
1169  auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64rr))
1170  .addReg(TargetReg, RegState::Kill)
1171  .addReg(AddrReg, RegState::Kill);
1172  ++NumInstsInserted;
1173  (void)CheckI;
1174  LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
1175  }
1176 
1177  // Now cmov over the predicate if the comparison wasn't equal.
1178  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
1179  auto CMovOp = X86::getCMovFromCond(X86::COND_NE, PredStateSizeInBytes);
1180  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
1181  auto CMovI =
1182  BuildMI(MBB, InsertPt, DebugLoc(), TII->get(CMovOp), UpdatedStateReg)
1183  .addReg(PS->InitialReg)
1184  .addReg(PS->PoisonReg);
1185  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
1186  ++NumInstsInserted;
1187  LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
1188  CMovs.push_back(&*CMovI);
1189 
1190  // And put the new value into the available values for SSA form of our
1191  // predicate state.
1192  PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
1193  }
1194 
1195  // Return all the newly inserted cmov instructions of the predicate state.
1196  return CMovs;
1197 }
1198 
1199 /// Returns true if the instruction has no behavior (specified or otherwise)
1200 /// that is based on the value of any of its register operands
1201 ///
1202 /// A classical example of something that is inherently not data invariant is an
1203 /// indirect jump -- the destination is loaded into icache based on the bits set
1204 /// in the jump destination register.
1205 ///
1206 /// FIXME: This should become part of our instruction tables.
1208  switch (MI.getOpcode()) {
1209  default:
1210  // By default, assume that the instruction is not data invariant.
1211  return false;
1212 
1213  // Some target-independent operations that trivially lower to data-invariant
1214  // instructions.
1215  case TargetOpcode::COPY:
1216  case TargetOpcode::INSERT_SUBREG:
1217  case TargetOpcode::SUBREG_TO_REG:
1218  return true;
1219 
1220  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
1221  // However, they set flags and are perhaps the most surprisingly constant
1222  // time operations so we call them out here separately.
1223  case X86::IMUL16rr:
1224  case X86::IMUL16rri8:
1225  case X86::IMUL16rri:
1226  case X86::IMUL32rr:
1227  case X86::IMUL32rri8:
1228  case X86::IMUL32rri:
1229  case X86::IMUL64rr:
1230  case X86::IMUL64rri32:
1231  case X86::IMUL64rri8:
1232 
1233  // Bit scanning and counting instructions that are somewhat surprisingly
1234  // constant time as they scan across bits and do other fairly complex
1235  // operations like popcnt, but are believed to be constant time on x86.
1236  // However, these set flags.
1237  case X86::BSF16rr:
1238  case X86::BSF32rr:
1239  case X86::BSF64rr:
1240  case X86::BSR16rr:
1241  case X86::BSR32rr:
1242  case X86::BSR64rr:
1243  case X86::LZCNT16rr:
1244  case X86::LZCNT32rr:
1245  case X86::LZCNT64rr:
1246  case X86::POPCNT16rr:
1247  case X86::POPCNT32rr:
1248  case X86::POPCNT64rr:
1249  case X86::TZCNT16rr:
1250  case X86::TZCNT32rr:
1251  case X86::TZCNT64rr:
1252 
1253  // Bit manipulation instructions are effectively combinations of basic
1254  // arithmetic ops, and should still execute in constant time. These also
1255  // set flags.
1256  case X86::BLCFILL32rr:
1257  case X86::BLCFILL64rr:
1258  case X86::BLCI32rr:
1259  case X86::BLCI64rr:
1260  case X86::BLCIC32rr:
1261  case X86::BLCIC64rr:
1262  case X86::BLCMSK32rr:
1263  case X86::BLCMSK64rr:
1264  case X86::BLCS32rr:
1265  case X86::BLCS64rr:
1266  case X86::BLSFILL32rr:
1267  case X86::BLSFILL64rr:
1268  case X86::BLSI32rr:
1269  case X86::BLSI64rr:
1270  case X86::BLSIC32rr:
1271  case X86::BLSIC64rr:
1272  case X86::BLSMSK32rr:
1273  case X86::BLSMSK64rr:
1274  case X86::BLSR32rr:
1275  case X86::BLSR64rr:
1276  case X86::TZMSK32rr:
1277  case X86::TZMSK64rr:
1278 
1279  // Bit extracting and clearing instructions should execute in constant time,
1280  // and set flags.
1281  case X86::BEXTR32rr:
1282  case X86::BEXTR64rr:
1283  case X86::BEXTRI32ri:
1284  case X86::BEXTRI64ri:
1285  case X86::BZHI32rr:
1286  case X86::BZHI64rr:
1287 
1288  // Shift and rotate.
1289  case X86::ROL8r1: case X86::ROL16r1: case X86::ROL32r1: case X86::ROL64r1:
1290  case X86::ROL8rCL: case X86::ROL16rCL: case X86::ROL32rCL: case X86::ROL64rCL:
1291  case X86::ROL8ri: case X86::ROL16ri: case X86::ROL32ri: case X86::ROL64ri:
1292  case X86::ROR8r1: case X86::ROR16r1: case X86::ROR32r1: case X86::ROR64r1:
1293  case X86::ROR8rCL: case X86::ROR16rCL: case X86::ROR32rCL: case X86::ROR64rCL:
1294  case X86::ROR8ri: case X86::ROR16ri: case X86::ROR32ri: case X86::ROR64ri:
1295  case X86::SAR8r1: case X86::SAR16r1: case X86::SAR32r1: case X86::SAR64r1:
1296  case X86::SAR8rCL: case X86::SAR16rCL: case X86::SAR32rCL: case X86::SAR64rCL:
1297  case X86::SAR8ri: case X86::SAR16ri: case X86::SAR32ri: case X86::SAR64ri:
1298  case X86::SHL8r1: case X86::SHL16r1: case X86::SHL32r1: case X86::SHL64r1:
1299  case X86::SHL8rCL: case X86::SHL16rCL: case X86::SHL32rCL: case X86::SHL64rCL:
1300  case X86::SHL8ri: case X86::SHL16ri: case X86::SHL32ri: case X86::SHL64ri:
1301  case X86::SHR8r1: case X86::SHR16r1: case X86::SHR32r1: case X86::SHR64r1:
1302  case X86::SHR8rCL: case X86::SHR16rCL: case X86::SHR32rCL: case X86::SHR64rCL:
1303  case X86::SHR8ri: case X86::SHR16ri: case X86::SHR32ri: case X86::SHR64ri:
1304  case X86::SHLD16rrCL: case X86::SHLD32rrCL: case X86::SHLD64rrCL:
1305  case X86::SHLD16rri8: case X86::SHLD32rri8: case X86::SHLD64rri8:
1306  case X86::SHRD16rrCL: case X86::SHRD32rrCL: case X86::SHRD64rrCL:
1307  case X86::SHRD16rri8: case X86::SHRD32rri8: case X86::SHRD64rri8:
1308 
1309  // Basic arithmetic is constant time on the input but does set flags.
1310  case X86::ADC8rr: case X86::ADC8ri:
1311  case X86::ADC16rr: case X86::ADC16ri: case X86::ADC16ri8:
1312  case X86::ADC32rr: case X86::ADC32ri: case X86::ADC32ri8:
1313  case X86::ADC64rr: case X86::ADC64ri8: case X86::ADC64ri32:
1314  case X86::ADD8rr: case X86::ADD8ri:
1315  case X86::ADD16rr: case X86::ADD16ri: case X86::ADD16ri8:
1316  case X86::ADD32rr: case X86::ADD32ri: case X86::ADD32ri8:
1317  case X86::ADD64rr: case X86::ADD64ri8: case X86::ADD64ri32:
1318  case X86::AND8rr: case X86::AND8ri:
1319  case X86::AND16rr: case X86::AND16ri: case X86::AND16ri8:
1320  case X86::AND32rr: case X86::AND32ri: case X86::AND32ri8:
1321  case X86::AND64rr: case X86::AND64ri8: case X86::AND64ri32:
1322  case X86::OR8rr: case X86::OR8ri:
1323  case X86::OR16rr: case X86::OR16ri: case X86::OR16ri8:
1324  case X86::OR32rr: case X86::OR32ri: case X86::OR32ri8:
1325  case X86::OR64rr: case X86::OR64ri8: case X86::OR64ri32:
1326  case X86::SBB8rr: case X86::SBB8ri:
1327  case X86::SBB16rr: case X86::SBB16ri: case X86::SBB16ri8:
1328  case X86::SBB32rr: case X86::SBB32ri: case X86::SBB32ri8:
1329  case X86::SBB64rr: case X86::SBB64ri8: case X86::SBB64ri32:
1330  case X86::SUB8rr: case X86::SUB8ri:
1331  case X86::SUB16rr: case X86::SUB16ri: case X86::SUB16ri8:
1332  case X86::SUB32rr: case X86::SUB32ri: case X86::SUB32ri8:
1333  case X86::SUB64rr: case X86::SUB64ri8: case X86::SUB64ri32:
1334  case X86::XOR8rr: case X86::XOR8ri:
1335  case X86::XOR16rr: case X86::XOR16ri: case X86::XOR16ri8:
1336  case X86::XOR32rr: case X86::XOR32ri: case X86::XOR32ri8:
1337  case X86::XOR64rr: case X86::XOR64ri8: case X86::XOR64ri32:
1338  // Arithmetic with just 32-bit and 64-bit variants and no immediates.
1339  case X86::ADCX32rr: case X86::ADCX64rr:
1340  case X86::ADOX32rr: case X86::ADOX64rr:
1341  case X86::ANDN32rr: case X86::ANDN64rr:
1342  // Unary arithmetic operations.
1343  case X86::DEC8r: case X86::DEC16r: case X86::DEC32r: case X86::DEC64r:
1344  case X86::INC8r: case X86::INC16r: case X86::INC32r: case X86::INC64r:
1345  case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r:
1346  // Check whether the EFLAGS implicit-def is dead. We assume that this will
1347  // always find the implicit-def because this code should only be reached
1348  // for instructions that do in fact implicitly def this.
1349  if (!MI.findRegisterDefOperand(X86::EFLAGS)->isDead()) {
1350  // If we would clobber EFLAGS that are used, just bail for now.
1351  LLVM_DEBUG(dbgs() << " Unable to harden post-load due to EFLAGS: ";
1352  MI.dump(); dbgs() << "\n");
1353  return false;
1354  }
1355 
1356  // Otherwise, fallthrough to handle these the same as instructions that
1357  // don't set EFLAGS.
1359 
1360  // Unlike other arithmetic, NOT doesn't set EFLAGS.
1361  case X86::NOT8r: case X86::NOT16r: case X86::NOT32r: case X86::NOT64r:
1362 
1363  // Various move instructions used to zero or sign extend things. Note that we
1364  // intentionally don't support the _NOREX variants as we can't handle that
1365  // register constraint anyways.
1366  case X86::MOVSX16rr8:
1367  case X86::MOVSX32rr8: case X86::MOVSX32rr16:
1368  case X86::MOVSX64rr8: case X86::MOVSX64rr16: case X86::MOVSX64rr32:
1369  case X86::MOVZX16rr8:
1370  case X86::MOVZX32rr8: case X86::MOVZX32rr16:
1371  case X86::MOVZX64rr8: case X86::MOVZX64rr16:
1372  case X86::MOV32rr:
1373 
1374  // Arithmetic instructions that are both constant time and don't set flags.
1375  case X86::RORX32ri:
1376  case X86::RORX64ri:
1377  case X86::SARX32rr:
1378  case X86::SARX64rr:
1379  case X86::SHLX32rr:
1380  case X86::SHLX64rr:
1381  case X86::SHRX32rr:
1382  case X86::SHRX64rr:
1383 
1384  // LEA doesn't actually access memory, and its arithmetic is constant time.
1385  case X86::LEA16r:
1386  case X86::LEA32r:
1387  case X86::LEA64_32r:
1388  case X86::LEA64r:
1389  return true;
1390  }
1391 }
1392 
1393 /// Returns true if the instruction has no behavior (specified or otherwise)
1394 /// that is based on the value loaded from memory or the value of any
1395 /// non-address register operands.
1396 ///
1397 /// For example, if the latency of the instruction is dependent on the
1398 /// particular bits set in any of the registers *or* any of the bits loaded from
1399 /// memory.
1400 ///
1401 /// A classical example of something that is inherently not data invariant is an
1402 /// indirect jump -- the destination is loaded into icache based on the bits set
1403 /// in the jump destination register.
1404 ///
1405 /// FIXME: This should become part of our instruction tables.
1407  switch (MI.getOpcode()) {
1408  default:
1409  // By default, assume that the load will immediately leak.
1410  return false;
1411 
1412  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
1413  // However, they set flags and are perhaps the most surprisingly constant
1414  // time operations so we call them out here separately.
1415  case X86::IMUL16rm:
1416  case X86::IMUL16rmi8:
1417  case X86::IMUL16rmi:
1418  case X86::IMUL32rm:
1419  case X86::IMUL32rmi8:
1420  case X86::IMUL32rmi:
1421  case X86::IMUL64rm:
1422  case X86::IMUL64rmi32:
1423  case X86::IMUL64rmi8:
1424 
1425  // Bit scanning and counting instructions that are somewhat surprisingly
1426  // constant time as they scan across bits and do other fairly complex
1427  // operations like popcnt, but are believed to be constant time on x86.
1428  // However, these set flags.
1429  case X86::BSF16rm:
1430  case X86::BSF32rm:
1431  case X86::BSF64rm:
1432  case X86::BSR16rm:
1433  case X86::BSR32rm:
1434  case X86::BSR64rm:
1435  case X86::LZCNT16rm:
1436  case X86::LZCNT32rm:
1437  case X86::LZCNT64rm:
1438  case X86::POPCNT16rm:
1439  case X86::POPCNT32rm:
1440  case X86::POPCNT64rm:
1441  case X86::TZCNT16rm:
1442  case X86::TZCNT32rm:
1443  case X86::TZCNT64rm:
1444 
1445  // Bit manipulation instructions are effectively combinations of basic
1446  // arithmetic ops, and should still execute in constant time. These also
1447  // set flags.
1448  case X86::BLCFILL32rm:
1449  case X86::BLCFILL64rm:
1450  case X86::BLCI32rm:
1451  case X86::BLCI64rm:
1452  case X86::BLCIC32rm:
1453  case X86::BLCIC64rm:
1454  case X86::BLCMSK32rm:
1455  case X86::BLCMSK64rm:
1456  case X86::BLCS32rm:
1457  case X86::BLCS64rm:
1458  case X86::BLSFILL32rm:
1459  case X86::BLSFILL64rm:
1460  case X86::BLSI32rm:
1461  case X86::BLSI64rm:
1462  case X86::BLSIC32rm:
1463  case X86::BLSIC64rm:
1464  case X86::BLSMSK32rm:
1465  case X86::BLSMSK64rm:
1466  case X86::BLSR32rm:
1467  case X86::BLSR64rm:
1468  case X86::TZMSK32rm:
1469  case X86::TZMSK64rm:
1470 
1471  // Bit extracting and clearing instructions should execute in constant time,
1472  // and set flags.
1473  case X86::BEXTR32rm:
1474  case X86::BEXTR64rm:
1475  case X86::BEXTRI32mi:
1476  case X86::BEXTRI64mi:
1477  case X86::BZHI32rm:
1478  case X86::BZHI64rm:
1479 
1480  // Basic arithmetic is constant time on the input but does set flags.
1481  case X86::ADC8rm:
1482  case X86::ADC16rm:
1483  case X86::ADC32rm:
1484  case X86::ADC64rm:
1485  case X86::ADCX32rm:
1486  case X86::ADCX64rm:
1487  case X86::ADD8rm:
1488  case X86::ADD16rm:
1489  case X86::ADD32rm:
1490  case X86::ADD64rm:
1491  case X86::ADOX32rm:
1492  case X86::ADOX64rm:
1493  case X86::AND8rm:
1494  case X86::AND16rm:
1495  case X86::AND32rm:
1496  case X86::AND64rm:
1497  case X86::ANDN32rm:
1498  case X86::ANDN64rm:
1499  case X86::OR8rm:
1500  case X86::OR16rm:
1501  case X86::OR32rm:
1502  case X86::OR64rm:
1503  case X86::SBB8rm:
1504  case X86::SBB16rm:
1505  case X86::SBB32rm:
1506  case X86::SBB64rm:
1507  case X86::SUB8rm:
1508  case X86::SUB16rm:
1509  case X86::SUB32rm:
1510  case X86::SUB64rm:
1511  case X86::XOR8rm:
1512  case X86::XOR16rm:
1513  case X86::XOR32rm:
1514  case X86::XOR64rm:
1515  // Check whether the EFLAGS implicit-def is dead. We assume that this will
1516  // always find the implicit-def because this code should only be reached
1517  // for instructions that do in fact implicitly def this.
1518  if (!MI.findRegisterDefOperand(X86::EFLAGS)->isDead()) {
1519  // If we would clobber EFLAGS that are used, just bail for now.
1520  LLVM_DEBUG(dbgs() << " Unable to harden post-load due to EFLAGS: ";
1521  MI.dump(); dbgs() << "\n");
1522  return false;
1523  }
1524 
1525  // Otherwise, fallthrough to handle these the same as instructions that
1526  // don't set EFLAGS.
1528 
1529  // Integer multiply w/o affecting flags is still believed to be constant
1530  // time on x86. Called out separately as this is among the most surprising
1531  // instructions to exhibit that behavior.
1532  case X86::MULX32rm:
1533  case X86::MULX64rm:
1534 
1535  // Arithmetic instructions that are both constant time and don't set flags.
1536  case X86::RORX32mi:
1537  case X86::RORX64mi:
1538  case X86::SARX32rm:
1539  case X86::SARX64rm:
1540  case X86::SHLX32rm:
1541  case X86::SHLX64rm:
1542  case X86::SHRX32rm:
1543  case X86::SHRX64rm:
1544 
1545  // Conversions are believed to be constant time and don't set flags.
1546  case X86::CVTTSD2SI64rm: case X86::VCVTTSD2SI64rm: case X86::VCVTTSD2SI64Zrm:
1547  case X86::CVTTSD2SIrm: case X86::VCVTTSD2SIrm: case X86::VCVTTSD2SIZrm:
1548  case X86::CVTTSS2SI64rm: case X86::VCVTTSS2SI64rm: case X86::VCVTTSS2SI64Zrm:
1549  case X86::CVTTSS2SIrm: case X86::VCVTTSS2SIrm: case X86::VCVTTSS2SIZrm:
1550  case X86::CVTSI2SDrm: case X86::VCVTSI2SDrm: case X86::VCVTSI2SDZrm:
1551  case X86::CVTSI2SSrm: case X86::VCVTSI2SSrm: case X86::VCVTSI2SSZrm:
1552  case X86::CVTSI642SDrm: case X86::VCVTSI642SDrm: case X86::VCVTSI642SDZrm:
1553  case X86::CVTSI642SSrm: case X86::VCVTSI642SSrm: case X86::VCVTSI642SSZrm:
1554  case X86::CVTSS2SDrm: case X86::VCVTSS2SDrm: case X86::VCVTSS2SDZrm:
1555  case X86::CVTSD2SSrm: case X86::VCVTSD2SSrm: case X86::VCVTSD2SSZrm:
1556  // AVX512 added unsigned integer conversions.
1557  case X86::VCVTTSD2USI64Zrm:
1558  case X86::VCVTTSD2USIZrm:
1559  case X86::VCVTTSS2USI64Zrm:
1560  case X86::VCVTTSS2USIZrm:
1561  case X86::VCVTUSI2SDZrm:
1562  case X86::VCVTUSI642SDZrm:
1563  case X86::VCVTUSI2SSZrm:
1564  case X86::VCVTUSI642SSZrm:
1565 
1566  // Loads to register don't set flags.
1567  case X86::MOV8rm:
1568  case X86::MOV8rm_NOREX:
1569  case X86::MOV16rm:
1570  case X86::MOV32rm:
1571  case X86::MOV64rm:
1572  case X86::MOVSX16rm8:
1573  case X86::MOVSX32rm16:
1574  case X86::MOVSX32rm8:
1575  case X86::MOVSX32rm8_NOREX:
1576  case X86::MOVSX64rm16:
1577  case X86::MOVSX64rm32:
1578  case X86::MOVSX64rm8:
1579  case X86::MOVZX16rm8:
1580  case X86::MOVZX32rm16:
1581  case X86::MOVZX32rm8:
1582  case X86::MOVZX32rm8_NOREX:
1583  case X86::MOVZX64rm16:
1584  case X86::MOVZX64rm8:
1585  return true;
1586  }
1587 }
1588 
1590  const TargetRegisterInfo &TRI) {
1591  // Check if EFLAGS are alive by seeing if there is a def of them or they
1592  // live-in, and then seeing if that def is in turn used.
1593  for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) {
1594  if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) {
1595  // If the def is dead, then EFLAGS is not live.
1596  if (DefOp->isDead())
1597  return false;
1598 
1599  // Otherwise we've def'ed it, and it is live.
1600  return true;
1601  }
1602  // While at this instruction, also check if we use and kill EFLAGS
1603  // which means it isn't live.
1604  if (MI.killsRegister(X86::EFLAGS, &TRI))
1605  return false;
1606  }
1607 
1608  // If we didn't find anything conclusive (neither definitely alive or
1609  // definitely dead) return whether it lives into the block.
1610  return MBB.isLiveIn(X86::EFLAGS);
1611 }
1612 
1613 /// Trace the predicate state through each of the blocks in the function,
1614 /// hardening everything necessary along the way.
1615 ///
1616 /// We call this routine once the initial predicate state has been established
1617 /// for each basic block in the function in the SSA updater. This routine traces
1618 /// it through the instructions within each basic block, and for non-returning
1619 /// blocks informs the SSA updater about the final state that lives out of the
1620 /// block. Along the way, it hardens any vulnerable instruction using the
1621 /// currently valid predicate state. We have to do these two things together
1622 /// because the SSA updater only works across blocks. Within a block, we track
1623 /// the current predicate state directly and update it as it changes.
1624 ///
1625 /// This operates in two passes over each block. First, we analyze the loads in
1626 /// the block to determine which strategy will be used to harden them: hardening
1627 /// the address or hardening the loaded value when loaded into a register
1628 /// amenable to hardening. We have to process these first because the two
1629 /// strategies may interact -- later hardening may change what strategy we wish
1630 /// to use. We also will analyze data dependencies between loads and avoid
1631 /// hardening those loads that are data dependent on a load with a hardened
1632 /// address. We also skip hardening loads already behind an LFENCE as that is
1633 /// sufficient to harden them against misspeculation.
1634 ///
1635 /// Second, we actively trace the predicate state through the block, applying
1636 /// the hardening steps we determined necessary in the first pass as we go.
1637 ///
1638 /// These two passes are applied to each basic block. We operate one block at a
1639 /// time to simplify reasoning about reachability and sequencing.
1640 void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(
1641  MachineFunction &MF) {
1642  SmallPtrSet<MachineInstr *, 16> HardenPostLoad;
1643  SmallPtrSet<MachineInstr *, 16> HardenLoadAddr;
1644 
1645  SmallSet<unsigned, 16> HardenedAddrRegs;
1646 
1647  SmallDenseMap<unsigned, unsigned, 32> AddrRegToHardenedReg;
1648 
1649  // Track the set of load-dependent registers through the basic block. Because
1650  // the values of these registers have an existing data dependency on a loaded
1651  // value which we would have checked, we can omit any checks on them.
1652  SparseBitVector<> LoadDepRegs;
1653 
1654  for (MachineBasicBlock &MBB : MF) {
1655  // The first pass over the block: collect all the loads which can have their
1656  // loaded value hardened and all the loads that instead need their address
1657  // hardened. During this walk we propagate load dependence for address
1658  // hardened loads and also look for LFENCE to stop hardening wherever
1659  // possible. When deciding whether or not to harden the loaded value or not,
1660  // we check to see if any registers used in the address will have been
1661  // hardened at this point and if so, harden any remaining address registers
1662  // as that often successfully re-uses hardened addresses and minimizes
1663  // instructions.
1664  //
1665  // FIXME: We should consider an aggressive mode where we continue to keep as
1666  // many loads value hardened even when some address register hardening would
1667  // be free (due to reuse).
1668  //
1669  // Note that we only need this pass if we are actually hardening loads.
1670  if (HardenLoads)
1671  for (MachineInstr &MI : MBB) {
1672  // We naively assume that all def'ed registers of an instruction have
1673  // a data dependency on all of their operands.
1674  // FIXME: Do a more careful analysis of x86 to build a conservative
1675  // model here.
1676  if (llvm::any_of(MI.uses(), [&](MachineOperand &Op) {
1677  return Op.isReg() && LoadDepRegs.test(Op.getReg());
1678  }))
1679  for (MachineOperand &Def : MI.defs())
1680  if (Def.isReg())
1681  LoadDepRegs.set(Def.getReg());
1682 
1683  // Both Intel and AMD are guiding that they will change the semantics of
1684  // LFENCE to be a speculation barrier, so if we see an LFENCE, there is
1685  // no more need to guard things in this block.
1686  if (MI.getOpcode() == X86::LFENCE)
1687  break;
1688 
1689  // If this instruction cannot load, nothing to do.
1690  if (!MI.mayLoad())
1691  continue;
1692 
1693  // Some instructions which "load" are trivially safe or unimportant.
1694  if (MI.getOpcode() == X86::MFENCE)
1695  continue;
1696 
1697  // Extract the memory operand information about this instruction.
1698  // FIXME: This doesn't handle loading pseudo instructions which we often
1699  // could handle with similarly generic logic. We probably need to add an
1700  // MI-layer routine similar to the MC-layer one we use here which maps
1701  // pseudos much like this maps real instructions.
1702  const MCInstrDesc &Desc = MI.getDesc();
1703  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
1704  if (MemRefBeginIdx < 0) {
1705  LLVM_DEBUG(dbgs()
1706  << "WARNING: unable to harden loading instruction: ";
1707  MI.dump());
1708  continue;
1709  }
1710 
1711  MemRefBeginIdx += X86II::getOperandBias(Desc);
1712 
1713  MachineOperand &BaseMO =
1714  MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
1715  MachineOperand &IndexMO =
1716  MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
1717 
1718  // If we have at least one (non-frame-index, non-RIP) register operand,
1719  // and neither operand is load-dependent, we need to check the load.
1720  unsigned BaseReg = 0, IndexReg = 0;
1721  if (!BaseMO.isFI() && BaseMO.getReg() != X86::RIP &&
1722  BaseMO.getReg() != X86::NoRegister)
1723  BaseReg = BaseMO.getReg();
1724  if (IndexMO.getReg() != X86::NoRegister)
1725  IndexReg = IndexMO.getReg();
1726 
1727  if (!BaseReg && !IndexReg)
1728  // No register operands!
1729  continue;
1730 
1731  // If any register operand is dependent, this load is dependent and we
1732  // needn't check it.
1733  // FIXME: Is this true in the case where we are hardening loads after
1734  // they complete? Unclear, need to investigate.
1735  if ((BaseReg && LoadDepRegs.test(BaseReg)) ||
1736  (IndexReg && LoadDepRegs.test(IndexReg)))
1737  continue;
1738 
1739  // If post-load hardening is enabled, this load is compatible with
1740  // post-load hardening, and we aren't already going to harden one of the
1741  // address registers, queue it up to be hardened post-load. Notably,
1742  // even once hardened this won't introduce a useful dependency that
1743  // could prune out subsequent loads.
1745  MI.getDesc().getNumDefs() == 1 && MI.getOperand(0).isReg() &&
1746  canHardenRegister(MI.getOperand(0).getReg()) &&
1747  !HardenedAddrRegs.count(BaseReg) &&
1748  !HardenedAddrRegs.count(IndexReg)) {
1749  HardenPostLoad.insert(&MI);
1750  HardenedAddrRegs.insert(MI.getOperand(0).getReg());
1751  continue;
1752  }
1753 
1754  // Record this instruction for address hardening and record its register
1755  // operands as being address-hardened.
1756  HardenLoadAddr.insert(&MI);
1757  if (BaseReg)
1758  HardenedAddrRegs.insert(BaseReg);
1759  if (IndexReg)
1760  HardenedAddrRegs.insert(IndexReg);
1761 
1762  for (MachineOperand &Def : MI.defs())
1763  if (Def.isReg())
1764  LoadDepRegs.set(Def.getReg());
1765  }
1766 
1767  // Now re-walk the instructions in the basic block, and apply whichever
1768  // hardening strategy we have elected. Note that we do this in a second
1769  // pass specifically so that we have the complete set of instructions for
1770  // which we will do post-load hardening and can defer it in certain
1771  // circumstances.
1772  for (MachineInstr &MI : MBB) {
1773  if (HardenLoads) {
1774  // We cannot both require hardening the def of a load and its address.
1775  assert(!(HardenLoadAddr.count(&MI) && HardenPostLoad.count(&MI)) &&
1776  "Requested to harden both the address and def of a load!");
1777 
1778  // Check if this is a load whose address needs to be hardened.
1779  if (HardenLoadAddr.erase(&MI)) {
1780  const MCInstrDesc &Desc = MI.getDesc();
1781  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
1782  assert(MemRefBeginIdx >= 0 && "Cannot have an invalid index here!");
1783 
1784  MemRefBeginIdx += X86II::getOperandBias(Desc);
1785 
1786  MachineOperand &BaseMO =
1787  MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
1788  MachineOperand &IndexMO =
1789  MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
1790  hardenLoadAddr(MI, BaseMO, IndexMO, AddrRegToHardenedReg);
1791  continue;
1792  }
1793 
1794  // Test if this instruction is one of our post load instructions (and
1795  // remove it from the set if so).
1796  if (HardenPostLoad.erase(&MI)) {
1797  assert(!MI.isCall() && "Must not try to post-load harden a call!");
1798 
1799  // If this is a data-invariant load, we want to try and sink any
1800  // hardening as far as possible.
1801  if (isDataInvariantLoad(MI)) {
1802  // Sink the instruction we'll need to harden as far as we can down
1803  // the graph.
1804  MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad);
1805 
1806  // If we managed to sink this instruction, update everything so we
1807  // harden that instruction when we reach it in the instruction
1808  // sequence.
1809  if (SunkMI != &MI) {
1810  // If in sinking there was no instruction needing to be hardened,
1811  // we're done.
1812  if (!SunkMI)
1813  continue;
1814 
1815  // Otherwise, add this to the set of defs we harden.
1816  HardenPostLoad.insert(SunkMI);
1817  continue;
1818  }
1819  }
1820 
1821  unsigned HardenedReg = hardenPostLoad(MI);
1822 
1823  // Mark the resulting hardened register as such so we don't re-harden.
1824  AddrRegToHardenedReg[HardenedReg] = HardenedReg;
1825 
1826  continue;
1827  }
1828 
1829  // Check for an indirect call or branch that may need its input hardened
1830  // even if we couldn't find the specific load used, or were able to
1831  // avoid hardening it for some reason. Note that here we cannot break
1832  // out afterward as we may still need to handle any call aspect of this
1833  // instruction.
1834  if ((MI.isCall() || MI.isBranch()) && HardenIndirectCallsAndJumps)
1835  hardenIndirectCallOrJumpInstr(MI, AddrRegToHardenedReg);
1836  }
1837 
1838  // After we finish hardening loads we handle interprocedural hardening if
1839  // enabled and relevant for this instruction.
1841  continue;
1842  if (!MI.isCall() && !MI.isReturn())
1843  continue;
1844 
1845  // If this is a direct return (IE, not a tail call) just directly harden
1846  // it.
1847  if (MI.isReturn() && !MI.isCall()) {
1848  hardenReturnInstr(MI);
1849  continue;
1850  }
1851 
1852  // Otherwise we have a call. We need to handle transferring the predicate
1853  // state into a call and recovering it after the call returns (unless this
1854  // is a tail call).
1855  assert(MI.isCall() && "Should only reach here for calls!");
1856  tracePredStateThroughCall(MI);
1857  }
1858 
1859  HardenPostLoad.clear();
1860  HardenLoadAddr.clear();
1861  HardenedAddrRegs.clear();
1862  AddrRegToHardenedReg.clear();
1863 
1864  // Currently, we only track data-dependent loads within a basic block.
1865  // FIXME: We should see if this is necessary or if we could be more
1866  // aggressive here without opening up attack avenues.
1867  LoadDepRegs.clear();
1868  }
1869 }
1870 
1871 /// Save EFLAGS into the returned GPR. This can in turn be restored with
1872 /// `restoreEFLAGS`.
1873 ///
1874 /// Note that LLVM can only lower very simple patterns of saved and restored
1875 /// EFLAGS registers. The restore should always be within the same basic block
1876 /// as the save so that no PHI nodes are inserted.
1877 unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS(
1879  DebugLoc Loc) {
1880  // FIXME: Hard coding this to a 32-bit register class seems weird, but matches
1881  // what instruction selection does.
1882  unsigned Reg = MRI->createVirtualRegister(&X86::GR32RegClass);
1883  // We directly copy the FLAGS register and rely on later lowering to clean
1884  // this up into the appropriate setCC instructions.
1885  BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), Reg).addReg(X86::EFLAGS);
1886  ++NumInstsInserted;
1887  return Reg;
1888 }
1889 
1890 /// Restore EFLAGS from the provided GPR. This should be produced by
1891 /// `saveEFLAGS`.
1892 ///
1893 /// This must be done within the same basic block as the save in order to
1894 /// reliably lower.
1895 void X86SpeculativeLoadHardeningPass::restoreEFLAGS(
1897  unsigned Reg) {
1898  BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), X86::EFLAGS).addReg(Reg);
1899  ++NumInstsInserted;
1900 }
1901 
1902 /// Takes the current predicate state (in a register) and merges it into the
1903 /// stack pointer. The state is essentially a single bit, but we merge this in
1904 /// a way that won't form non-canonical pointers and also will be preserved
1905 /// across normal stack adjustments.
1906 void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(
1908  unsigned PredStateReg) {
1909  unsigned TmpReg = MRI->createVirtualRegister(PS->RC);
1910  // FIXME: This hard codes a shift distance based on the number of bits needed
1911  // to stay canonical on 64-bit. We should compute this somehow and support
1912  // 32-bit as part of that.
1913  auto ShiftI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHL64ri), TmpReg)
1914  .addReg(PredStateReg, RegState::Kill)
1915  .addImm(47);
1916  ShiftI->addRegisterDead(X86::EFLAGS, TRI);
1917  ++NumInstsInserted;
1918  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), X86::RSP)
1919  .addReg(X86::RSP)
1920  .addReg(TmpReg, RegState::Kill);
1921  OrI->addRegisterDead(X86::EFLAGS, TRI);
1922  ++NumInstsInserted;
1923 }
1924 
1925 /// Extracts the predicate state stored in the high bits of the stack pointer.
1926 unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP(
1928  DebugLoc Loc) {
1929  unsigned PredStateReg = MRI->createVirtualRegister(PS->RC);
1930  unsigned TmpReg = MRI->createVirtualRegister(PS->RC);
1931 
1932  // We know that the stack pointer will have any preserved predicate state in
1933  // its high bit. We just want to smear this across the other bits. Turns out,
1934  // this is exactly what an arithmetic right shift does.
1935  BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), TmpReg)
1936  .addReg(X86::RSP);
1937  auto ShiftI =
1938  BuildMI(MBB, InsertPt, Loc, TII->get(X86::SAR64ri), PredStateReg)
1939  .addReg(TmpReg, RegState::Kill)
1940  .addImm(TRI->getRegSizeInBits(*PS->RC) - 1);
1941  ShiftI->addRegisterDead(X86::EFLAGS, TRI);
1942  ++NumInstsInserted;
1943 
1944  return PredStateReg;
1945 }
1946 
1947 void X86SpeculativeLoadHardeningPass::hardenLoadAddr(
1948  MachineInstr &MI, MachineOperand &BaseMO, MachineOperand &IndexMO,
1949  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
1950  MachineBasicBlock &MBB = *MI.getParent();
1951  DebugLoc Loc = MI.getDebugLoc();
1952 
1953  // Check if EFLAGS are alive by seeing if there is a def of them or they
1954  // live-in, and then seeing if that def is in turn used.
1955  bool EFLAGSLive = isEFLAGSLive(MBB, MI.getIterator(), *TRI);
1956 
1957  SmallVector<MachineOperand *, 2> HardenOpRegs;
1958 
1959  if (BaseMO.isFI()) {
1960  // A frame index is never a dynamically controllable load, so only
1961  // harden it if we're covering fixed address loads as well.
1962  LLVM_DEBUG(
1963  dbgs() << " Skipping hardening base of explicit stack frame load: ";
1964  MI.dump(); dbgs() << "\n");
1965  } else if (BaseMO.getReg() == X86::RIP ||
1966  BaseMO.getReg() == X86::NoRegister) {
1967  // For both RIP-relative addressed loads or absolute loads, we cannot
1968  // meaningfully harden them because the address being loaded has no
1969  // dynamic component.
1970  //
1971  // FIXME: When using a segment base (like TLS does) we end up with the
1972  // dynamic address being the base plus -1 because we can't mutate the
1973  // segment register here. This allows the signed 32-bit offset to point at
1974  // valid segment-relative addresses and load them successfully.
1975  LLVM_DEBUG(
1976  dbgs() << " Cannot harden base of "
1977  << (BaseMO.getReg() == X86::RIP ? "RIP-relative" : "no-base")
1978  << " address in a load!");
1979  } else {
1980  assert(BaseMO.isReg() &&
1981  "Only allowed to have a frame index or register base.");
1982  HardenOpRegs.push_back(&BaseMO);
1983  }
1984 
1985  if (IndexMO.getReg() != X86::NoRegister &&
1986  (HardenOpRegs.empty() ||
1987  HardenOpRegs.front()->getReg() != IndexMO.getReg()))
1988  HardenOpRegs.push_back(&IndexMO);
1989 
1990  assert((HardenOpRegs.size() == 1 || HardenOpRegs.size() == 2) &&
1991  "Should have exactly one or two registers to harden!");
1992  assert((HardenOpRegs.size() == 1 ||
1993  HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&
1994  "Should not have two of the same registers!");
1995 
1996  // Remove any registers that have alreaded been checked.
1997  llvm::erase_if(HardenOpRegs, [&](MachineOperand *Op) {
1998  // See if this operand's register has already been checked.
1999  auto It = AddrRegToHardenedReg.find(Op->getReg());
2000  if (It == AddrRegToHardenedReg.end())
2001  // Not checked, so retain this one.
2002  return false;
2003 
2004  // Otherwise, we can directly update this operand and remove it.
2005  Op->setReg(It->second);
2006  return true;
2007  });
2008  // If there are none left, we're done.
2009  if (HardenOpRegs.empty())
2010  return;
2011 
2012  // Compute the current predicate state.
2013  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2014 
2015  auto InsertPt = MI.getIterator();
2016 
2017  // If EFLAGS are live and we don't have access to instructions that avoid
2018  // clobbering EFLAGS we need to save and restore them. This in turn makes
2019  // the EFLAGS no longer live.
2020  unsigned FlagsReg = 0;
2021  if (EFLAGSLive && !Subtarget->hasBMI2()) {
2022  EFLAGSLive = false;
2023  FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
2024  }
2025 
2026  for (MachineOperand *Op : HardenOpRegs) {
2027  unsigned OpReg = Op->getReg();
2028  auto *OpRC = MRI->getRegClass(OpReg);
2029  unsigned TmpReg = MRI->createVirtualRegister(OpRC);
2030 
2031  // If this is a vector register, we'll need somewhat custom logic to handle
2032  // hardening it.
2033  if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) ||
2034  OpRC->hasSuperClassEq(&X86::VR256RegClass))) {
2035  assert(Subtarget->hasAVX2() && "AVX2-specific register classes!");
2036  bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);
2037 
2038  // Move our state into a vector register.
2039  // FIXME: We could skip this at the cost of longer encodings with AVX-512
2040  // but that doesn't seem likely worth it.
2041  unsigned VStateReg = MRI->createVirtualRegister(&X86::VR128RegClass);
2042  auto MovI =
2043  BuildMI(MBB, InsertPt, Loc, TII->get(X86::VMOV64toPQIrr), VStateReg)
2044  .addReg(StateReg);
2045  (void)MovI;
2046  ++NumInstsInserted;
2047  LLVM_DEBUG(dbgs() << " Inserting mov: "; MovI->dump(); dbgs() << "\n");
2048 
2049  // Broadcast it across the vector register.
2050  unsigned VBStateReg = MRI->createVirtualRegister(OpRC);
2051  auto BroadcastI = BuildMI(MBB, InsertPt, Loc,
2052  TII->get(Is128Bit ? X86::VPBROADCASTQrr
2053  : X86::VPBROADCASTQYrr),
2054  VBStateReg)
2055  .addReg(VStateReg);
2056  (void)BroadcastI;
2057  ++NumInstsInserted;
2058  LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();
2059  dbgs() << "\n");
2060 
2061  // Merge our potential poison state into the value with a vector or.
2062  auto OrI =
2063  BuildMI(MBB, InsertPt, Loc,
2064  TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg)
2065  .addReg(VBStateReg)
2066  .addReg(OpReg);
2067  (void)OrI;
2068  ++NumInstsInserted;
2069  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2070  } else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||
2071  OpRC->hasSuperClassEq(&X86::VR256XRegClass) ||
2072  OpRC->hasSuperClassEq(&X86::VR512RegClass)) {
2073  assert(Subtarget->hasAVX512() && "AVX512-specific register classes!");
2074  bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);
2075  bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);
2076  if (Is128Bit || Is256Bit)
2077  assert(Subtarget->hasVLX() && "AVX512VL-specific register classes!");
2078 
2079  // Broadcast our state into a vector register.
2080  unsigned VStateReg = MRI->createVirtualRegister(OpRC);
2081  unsigned BroadcastOp =
2082  Is128Bit ? X86::VPBROADCASTQrZ128r
2083  : Is256Bit ? X86::VPBROADCASTQrZ256r : X86::VPBROADCASTQrZr;
2084  auto BroadcastI =
2085  BuildMI(MBB, InsertPt, Loc, TII->get(BroadcastOp), VStateReg)
2086  .addReg(StateReg);
2087  (void)BroadcastI;
2088  ++NumInstsInserted;
2089  LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();
2090  dbgs() << "\n");
2091 
2092  // Merge our potential poison state into the value with a vector or.
2093  unsigned OrOp = Is128Bit ? X86::VPORQZ128rr
2094  : Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr;
2095  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOp), TmpReg)
2096  .addReg(VStateReg)
2097  .addReg(OpReg);
2098  (void)OrI;
2099  ++NumInstsInserted;
2100  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2101  } else {
2102  // FIXME: Need to support GR32 here for 32-bit code.
2103  assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&
2104  "Not a supported register class for address hardening!");
2105 
2106  if (!EFLAGSLive) {
2107  // Merge our potential poison state into the value with an or.
2108  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), TmpReg)
2109  .addReg(StateReg)
2110  .addReg(OpReg);
2111  OrI->addRegisterDead(X86::EFLAGS, TRI);
2112  ++NumInstsInserted;
2113  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2114  } else {
2115  // We need to avoid touching EFLAGS so shift out all but the least
2116  // significant bit using the instruction that doesn't update flags.
2117  auto ShiftI =
2118  BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHRX64rr), TmpReg)
2119  .addReg(OpReg)
2120  .addReg(StateReg);
2121  (void)ShiftI;
2122  ++NumInstsInserted;
2123  LLVM_DEBUG(dbgs() << " Inserting shrx: "; ShiftI->dump();
2124  dbgs() << "\n");
2125  }
2126  }
2127 
2128  // Record this register as checked and update the operand.
2129  assert(!AddrRegToHardenedReg.count(Op->getReg()) &&
2130  "Should not have checked this register yet!");
2131  AddrRegToHardenedReg[Op->getReg()] = TmpReg;
2132  Op->setReg(TmpReg);
2133  ++NumAddrRegsHardened;
2134  }
2135 
2136  // And restore the flags if needed.
2137  if (FlagsReg)
2138  restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
2139 }
2140 
2141 MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(
2142  MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) {
2143  assert(isDataInvariantLoad(InitialMI) &&
2144  "Cannot get here with a non-invariant load!");
2145 
2146  // See if we can sink hardening the loaded value.
2147  auto SinkCheckToSingleUse =
2149  unsigned DefReg = MI.getOperand(0).getReg();
2150 
2151  // We need to find a single use which we can sink the check. We can
2152  // primarily do this because many uses may already end up checked on their
2153  // own.
2154  MachineInstr *SingleUseMI = nullptr;
2155  for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) {
2156  // If we're already going to harden this use, it is data invariant and
2157  // within our block.
2158  if (HardenedInstrs.count(&UseMI)) {
2159  if (!isDataInvariantLoad(UseMI)) {
2160  // If we've already decided to harden a non-load, we must have sunk
2161  // some other post-load hardened instruction to it and it must itself
2162  // be data-invariant.
2164  "Data variant instruction being hardened!");
2165  continue;
2166  }
2167 
2168  // Otherwise, this is a load and the load component can't be data
2169  // invariant so check how this register is being used.
2170  const MCInstrDesc &Desc = UseMI.getDesc();
2171  int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
2172  assert(MemRefBeginIdx >= 0 &&
2173  "Should always have mem references here!");
2174  MemRefBeginIdx += X86II::getOperandBias(Desc);
2175 
2176  MachineOperand &BaseMO =
2177  UseMI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
2178  MachineOperand &IndexMO =
2179  UseMI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
2180  if ((BaseMO.isReg() && BaseMO.getReg() == DefReg) ||
2181  (IndexMO.isReg() && IndexMO.getReg() == DefReg))
2182  // The load uses the register as part of its address making it not
2183  // invariant.
2184  return {};
2185 
2186  continue;
2187  }
2188 
2189  if (SingleUseMI)
2190  // We already have a single use, this would make two. Bail.
2191  return {};
2192 
2193  // If this single use isn't data invariant, isn't in this block, or has
2194  // interfering EFLAGS, we can't sink the hardening to it.
2195  if (!isDataInvariant(UseMI) || UseMI.getParent() != MI.getParent())
2196  return {};
2197 
2198  // If this instruction defines multiple registers bail as we won't harden
2199  // all of them.
2200  if (UseMI.getDesc().getNumDefs() > 1)
2201  return {};
2202 
2203  // If this register isn't a virtual register we can't walk uses of sanely,
2204  // just bail. Also check that its register class is one of the ones we
2205  // can harden.
2206  unsigned UseDefReg = UseMI.getOperand(0).getReg();
2207  if (!TRI->isVirtualRegister(UseDefReg) ||
2208  !canHardenRegister(UseDefReg))
2209  return {};
2210 
2211  SingleUseMI = &UseMI;
2212  }
2213 
2214  // If SingleUseMI is still null, there is no use that needs its own
2215  // checking. Otherwise, it is the single use that needs checking.
2216  return {SingleUseMI};
2217  };
2218 
2219  MachineInstr *MI = &InitialMI;
2220  while (Optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*MI)) {
2221  // Update which MI we're checking now.
2222  MI = *SingleUse;
2223  if (!MI)
2224  break;
2225  }
2226 
2227  return MI;
2228 }
2229 
2230 bool X86SpeculativeLoadHardeningPass::canHardenRegister(unsigned Reg) {
2231  auto *RC = MRI->getRegClass(Reg);
2232  int RegBytes = TRI->getRegSizeInBits(*RC) / 8;
2233  if (RegBytes > 8)
2234  // We don't support post-load hardening of vectors.
2235  return false;
2236 
2237  // If this register class is explicitly constrained to a class that doesn't
2238  // require REX prefix, we may not be able to satisfy that constraint when
2239  // emitting the hardening instructions, so bail out here.
2240  // FIXME: This seems like a pretty lame hack. The way this comes up is when we
2241  // end up both with a NOREX and REX-only register as operands to the hardening
2242  // instructions. It would be better to fix that code to handle this situation
2243  // rather than hack around it in this way.
2244  const TargetRegisterClass *NOREXRegClasses[] = {
2245  &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,
2246  &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};
2247  if (RC == NOREXRegClasses[Log2_32(RegBytes)])
2248  return false;
2249 
2250  const TargetRegisterClass *GPRRegClasses[] = {
2251  &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,
2252  &X86::GR64RegClass};
2253  return RC->hasSuperClassEq(GPRRegClasses[Log2_32(RegBytes)]);
2254 }
2255 
2256 /// Harden a value in a register.
2257 ///
2258 /// This is the low-level logic to fully harden a value sitting in a register
2259 /// against leaking during speculative execution.
2260 ///
2261 /// Unlike hardening an address that is used by a load, this routine is required
2262 /// to hide *all* incoming bits in the register.
2263 ///
2264 /// `Reg` must be a virtual register. Currently, it is required to be a GPR no
2265 /// larger than the predicate state register. FIXME: We should support vector
2266 /// registers here by broadcasting the predicate state.
2267 ///
2268 /// The new, hardened virtual register is returned. It will have the same
2269 /// register class as `Reg`.
2270 unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister(
2271  unsigned Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
2272  DebugLoc Loc) {
2273  assert(canHardenRegister(Reg) && "Cannot harden this register!");
2274  assert(TRI->isVirtualRegister(Reg) && "Cannot harden a physical register!");
2275 
2276  auto *RC = MRI->getRegClass(Reg);
2277  int Bytes = TRI->getRegSizeInBits(*RC) / 8;
2278 
2279  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2280 
2281  // FIXME: Need to teach this about 32-bit mode.
2282  if (Bytes != 8) {
2283  unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};
2284  unsigned SubRegImm = SubRegImms[Log2_32(Bytes)];
2285  unsigned NarrowStateReg = MRI->createVirtualRegister(RC);
2286  BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), NarrowStateReg)
2287  .addReg(StateReg, 0, SubRegImm);
2288  StateReg = NarrowStateReg;
2289  }
2290 
2291  unsigned FlagsReg = 0;
2292  if (isEFLAGSLive(MBB, InsertPt, *TRI))
2293  FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
2294 
2295  unsigned NewReg = MRI->createVirtualRegister(RC);
2296  unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};
2297  unsigned OrOpCode = OrOpCodes[Log2_32(Bytes)];
2298  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOpCode), NewReg)
2299  .addReg(StateReg)
2300  .addReg(Reg);
2301  OrI->addRegisterDead(X86::EFLAGS, TRI);
2302  ++NumInstsInserted;
2303  LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");
2304 
2305  if (FlagsReg)
2306  restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
2307 
2308  return NewReg;
2309 }
2310 
2311 /// Harden a load by hardening the loaded value in the defined register.
2312 ///
2313 /// We can harden a non-leaking load into a register without touching the
2314 /// address by just hiding all of the loaded bits during misspeculation. We use
2315 /// an `or` instruction to do this because we set up our poison value as all
2316 /// ones. And the goal is just for the loaded bits to not be exposed to
2317 /// execution and coercing them to one is sufficient.
2318 ///
2319 /// Returns the newly hardened register.
2320 unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(MachineInstr &MI) {
2321  MachineBasicBlock &MBB = *MI.getParent();
2322  DebugLoc Loc = MI.getDebugLoc();
2323 
2324  auto &DefOp = MI.getOperand(0);
2325  unsigned OldDefReg = DefOp.getReg();
2326  auto *DefRC = MRI->getRegClass(OldDefReg);
2327 
2328  // Because we want to completely replace the uses of this def'ed value with
2329  // the hardened value, create a dedicated new register that will only be used
2330  // to communicate the unhardened value to the hardening.
2331  unsigned UnhardenedReg = MRI->createVirtualRegister(DefRC);
2332  DefOp.setReg(UnhardenedReg);
2333 
2334  // Now harden this register's value, getting a hardened reg that is safe to
2335  // use. Note that we insert the instructions to compute this *after* the
2336  // defining instruction, not before it.
2337  unsigned HardenedReg = hardenValueInRegister(
2338  UnhardenedReg, MBB, std::next(MI.getIterator()), Loc);
2339 
2340  // Finally, replace the old register (which now only has the uses of the
2341  // original def) with the hardened register.
2342  MRI->replaceRegWith(/*FromReg*/ OldDefReg, /*ToReg*/ HardenedReg);
2343 
2344  ++NumPostLoadRegsHardened;
2345  return HardenedReg;
2346 }
2347 
2348 /// Harden a return instruction.
2349 ///
2350 /// Returns implicitly perform a load which we need to harden. Without hardening
2351 /// this load, an attacker my speculatively write over the return address to
2352 /// steer speculation of the return to an attacker controlled address. This is
2353 /// called Spectre v1.1 or Bounds Check Bypass Store (BCBS) and is described in
2354 /// this paper:
2355 /// https://people.csail.mit.edu/vlk/spectre11.pdf
2356 ///
2357 /// We can harden this by introducing an LFENCE that will delay any load of the
2358 /// return address until prior instructions have retired (and thus are not being
2359 /// speculated), or we can harden the address used by the implicit load: the
2360 /// stack pointer.
2361 ///
2362 /// If we are not using an LFENCE, hardening the stack pointer has an additional
2363 /// benefit: it allows us to pass the predicate state accumulated in this
2364 /// function back to the caller. In the absence of a BCBS attack on the return,
2365 /// the caller will typically be resumed and speculatively executed due to the
2366 /// Return Stack Buffer (RSB) prediction which is very accurate and has a high
2367 /// priority. It is possible that some code from the caller will be executed
2368 /// speculatively even during a BCBS-attacked return until the steering takes
2369 /// effect. Whenever this happens, the caller can recover the (poisoned)
2370 /// predicate state from the stack pointer and continue to harden loads.
2371 void X86SpeculativeLoadHardeningPass::hardenReturnInstr(MachineInstr &MI) {
2372  MachineBasicBlock &MBB = *MI.getParent();
2373  DebugLoc Loc = MI.getDebugLoc();
2374  auto InsertPt = MI.getIterator();
2375 
2376  if (FenceCallAndRet)
2377  // No need to fence here as we'll fence at the return site itself. That
2378  // handles more cases than we can handle here.
2379  return;
2380 
2381  // Take our predicate state, shift it to the high 17 bits (so that we keep
2382  // pointers canonical) and merge it into RSP. This will allow the caller to
2383  // extract it when we return (speculatively).
2384  mergePredStateIntoSP(MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&MBB));
2385 }
2386 
2387 /// Trace the predicate state through a call.
2388 ///
2389 /// There are several layers of this needed to handle the full complexity of
2390 /// calls.
2391 ///
2392 /// First, we need to send the predicate state into the called function. We do
2393 /// this by merging it into the high bits of the stack pointer.
2394 ///
2395 /// For tail calls, this is all we need to do.
2396 ///
2397 /// For calls where we might return and resume the control flow, we need to
2398 /// extract the predicate state from the high bits of the stack pointer after
2399 /// control returns from the called function.
2400 ///
2401 /// We also need to verify that we intended to return to this location in the
2402 /// code. An attacker might arrange for the processor to mispredict the return
2403 /// to this valid but incorrect return address in the program rather than the
2404 /// correct one. See the paper on this attack, called "ret2spec" by the
2405 /// researchers, here:
2406 /// https://christian-rossow.de/publications/ret2spec-ccs2018.pdf
2407 ///
2408 /// The way we verify that we returned to the correct location is by preserving
2409 /// the expected return address across the call. One technique involves taking
2410 /// advantage of the red-zone to load the return address from `8(%rsp)` where it
2411 /// was left by the RET instruction when it popped `%rsp`. Alternatively, we can
2412 /// directly save the address into a register that will be preserved across the
2413 /// call. We compare this intended return address against the address
2414 /// immediately following the call (the observed return address). If these
2415 /// mismatch, we have detected misspeculation and can poison our predicate
2416 /// state.
2417 void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(
2418  MachineInstr &MI) {
2419  MachineBasicBlock &MBB = *MI.getParent();
2420  MachineFunction &MF = *MBB.getParent();
2421  auto InsertPt = MI.getIterator();
2422  DebugLoc Loc = MI.getDebugLoc();
2423 
2424  if (FenceCallAndRet) {
2425  if (MI.isReturn())
2426  // Tail call, we don't return to this function.
2427  // FIXME: We should also handle noreturn calls.
2428  return;
2429 
2430  // We don't need to fence before the call because the function should fence
2431  // in its entry. However, we do need to fence after the call returns.
2432  // Fencing before the return doesn't correctly handle cases where the return
2433  // itself is mispredicted.
2434  BuildMI(MBB, std::next(InsertPt), Loc, TII->get(X86::LFENCE));
2435  ++NumInstsInserted;
2436  ++NumLFENCEsInserted;
2437  return;
2438  }
2439 
2440  // First, we transfer the predicate state into the called function by merging
2441  // it into the stack pointer. This will kill the current def of the state.
2442  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2443  mergePredStateIntoSP(MBB, InsertPt, Loc, StateReg);
2444 
2445  // If this call is also a return, it is a tail call and we don't need anything
2446  // else to handle it so just return. Also, if there are no further
2447  // instructions and no successors, this call does not return so we can also
2448  // bail.
2449  if (MI.isReturn() || (std::next(InsertPt) == MBB.end() && MBB.succ_empty()))
2450  return;
2451 
2452  // Create a symbol to track the return address and attach it to the call
2453  // machine instruction. We will lower extra symbols attached to call
2454  // instructions as label immediately following the call.
2455  MCSymbol *RetSymbol =
2456  MF.getContext().createTempSymbol("slh_ret_addr",
2457  /*AlwaysAddSuffix*/ true);
2458  MI.setPostInstrSymbol(MF, RetSymbol);
2459 
2460  const TargetRegisterClass *AddrRC = &X86::GR64RegClass;
2461  unsigned ExpectedRetAddrReg = 0;
2462 
2463  // If we have no red zones or if the function returns twice (possibly without
2464  // using the `ret` instruction) like setjmp, we need to save the expected
2465  // return address prior to the call.
2466  if (MF.getFunction().hasFnAttribute(Attribute::NoRedZone) ||
2467  MF.exposesReturnsTwice()) {
2468  // If we don't have red zones, we need to compute the expected return
2469  // address prior to the call and store it in a register that lives across
2470  // the call.
2471  //
2472  // In some ways, this is doubly satisfying as a mitigation because it will
2473  // also successfully detect stack smashing bugs in some cases (typically,
2474  // when a callee-saved register is used and the callee doesn't push it onto
2475  // the stack). But that isn't our primary goal, so we only use it as
2476  // a fallback.
2477  //
2478  // FIXME: It isn't clear that this is reliable in the face of
2479  // rematerialization in the register allocator. We somehow need to force
2480  // that to not occur for this particular instruction, and instead to spill
2481  // or otherwise preserve the value computed *prior* to the call.
2482  //
2483  // FIXME: It is even less clear why MachineCSE can't just fold this when we
2484  // end up having to use identical instructions both before and after the
2485  // call to feed the comparison.
2486  ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
2487  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
2488  !Subtarget->isPositionIndependent()) {
2489  BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64ri32), ExpectedRetAddrReg)
2490  .addSym(RetSymbol);
2491  } else {
2492  BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ExpectedRetAddrReg)
2493  .addReg(/*Base*/ X86::RIP)
2494  .addImm(/*Scale*/ 1)
2495  .addReg(/*Index*/ 0)
2496  .addSym(RetSymbol)
2497  .addReg(/*Segment*/ 0);
2498  }
2499  }
2500 
2501  // Step past the call to handle when it returns.
2502  ++InsertPt;
2503 
2504  // If we didn't pre-compute the expected return address into a register, then
2505  // red zones are enabled and the return address is still available on the
2506  // stack immediately after the call. As the very first instruction, we load it
2507  // into a register.
2508  if (!ExpectedRetAddrReg) {
2509  ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
2510  BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64rm), ExpectedRetAddrReg)
2511  .addReg(/*Base*/ X86::RSP)
2512  .addImm(/*Scale*/ 1)
2513  .addReg(/*Index*/ 0)
2514  .addImm(/*Displacement*/ -8) // The stack pointer has been popped, so
2515  // the return address is 8-bytes past it.
2516  .addReg(/*Segment*/ 0);
2517  }
2518 
2519  // Now we extract the callee's predicate state from the stack pointer.
2520  unsigned NewStateReg = extractPredStateFromSP(MBB, InsertPt, Loc);
2521 
2522  // Test the expected return address against our actual address. If we can
2523  // form this basic block's address as an immediate, this is easy. Otherwise
2524  // we compute it.
2525  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
2526  !Subtarget->isPositionIndependent()) {
2527  // FIXME: Could we fold this with the load? It would require careful EFLAGS
2528  // management.
2529  BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64ri32))
2530  .addReg(ExpectedRetAddrReg, RegState::Kill)
2531  .addSym(RetSymbol);
2532  } else {
2533  unsigned ActualRetAddrReg = MRI->createVirtualRegister(AddrRC);
2534  BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ActualRetAddrReg)
2535  .addReg(/*Base*/ X86::RIP)
2536  .addImm(/*Scale*/ 1)
2537  .addReg(/*Index*/ 0)
2538  .addSym(RetSymbol)
2539  .addReg(/*Segment*/ 0);
2540  BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64rr))
2541  .addReg(ExpectedRetAddrReg, RegState::Kill)
2542  .addReg(ActualRetAddrReg, RegState::Kill);
2543  }
2544 
2545  // Now conditionally update the predicate state we just extracted if we ended
2546  // up at a different return address than expected.
2547  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
2548  auto CMovOp = X86::getCMovFromCond(X86::COND_NE, PredStateSizeInBytes);
2549 
2550  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
2551  auto CMovI = BuildMI(MBB, InsertPt, Loc, TII->get(CMovOp), UpdatedStateReg)
2552  .addReg(NewStateReg, RegState::Kill)
2553  .addReg(PS->PoisonReg);
2554  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
2555  ++NumInstsInserted;
2556  LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
2557 
2558  PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
2559 }
2560 
2561 /// An attacker may speculatively store over a value that is then speculatively
2562 /// loaded and used as the target of an indirect call or jump instruction. This
2563 /// is called Spectre v1.2 or Bounds Check Bypass Store (BCBS) and is described
2564 /// in this paper:
2565 /// https://people.csail.mit.edu/vlk/spectre11.pdf
2566 ///
2567 /// When this happens, the speculative execution of the call or jump will end up
2568 /// being steered to this attacker controlled address. While most such loads
2569 /// will be adequately hardened already, we want to ensure that they are
2570 /// definitively treated as needing post-load hardening. While address hardening
2571 /// is sufficient to prevent secret data from leaking to the attacker, it may
2572 /// not be sufficient to prevent an attacker from steering speculative
2573 /// execution. We forcibly unfolded all relevant loads above and so will always
2574 /// have an opportunity to post-load harden here, we just need to scan for cases
2575 /// not already flagged and add them.
2576 void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(
2577  MachineInstr &MI,
2578  SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
2579  switch (MI.getOpcode()) {
2580  case X86::FARCALL16m:
2581  case X86::FARCALL32m:
2582  case X86::FARCALL64:
2583  case X86::FARJMP16m:
2584  case X86::FARJMP32m:
2585  case X86::FARJMP64:
2586  // We don't need to harden either far calls or far jumps as they are
2587  // safe from Spectre.
2588  return;
2589 
2590  default:
2591  break;
2592  }
2593 
2594  // We should never see a loading instruction at this point, as those should
2595  // have been unfolded.
2596  assert(!MI.mayLoad() && "Found a lingering loading instruction!");
2597 
2598  // If the first operand isn't a register, this is a branch or call
2599  // instruction with an immediate operand which doesn't need to be hardened.
2600  if (!MI.getOperand(0).isReg())
2601  return;
2602 
2603  // For all of these, the target register is the first operand of the
2604  // instruction.
2605  auto &TargetOp = MI.getOperand(0);
2606  unsigned OldTargetReg = TargetOp.getReg();
2607 
2608  // Try to lookup a hardened version of this register. We retain a reference
2609  // here as we want to update the map to track any newly computed hardened
2610  // register.
2611  unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];
2612 
2613  // If we don't have a hardened register yet, compute one. Otherwise, just use
2614  // the already hardened register.
2615  //
2616  // FIXME: It is a little suspect that we use partially hardened registers that
2617  // only feed addresses. The complexity of partial hardening with SHRX
2618  // continues to pile up. Should definitively measure its value and consider
2619  // eliminating it.
2620  if (!HardenedTargetReg)
2621  HardenedTargetReg = hardenValueInRegister(
2622  OldTargetReg, *MI.getParent(), MI.getIterator(), MI.getDebugLoc());
2623 
2624  // Set the target operand to the hardened register.
2625  TargetOp.setReg(HardenedTargetReg);
2626 
2627  ++NumCallsOrJumpsHardened;
2628 }
2629 
2630 INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY,
2631  "X86 speculative load hardener", false, false)
2632 INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass, PASS_KEY,
2633  "X86 speculative load hardener", false, false)
2634 
2636  return new X86SpeculativeLoadHardeningPass();
2637 }
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
bool isCall(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:632
MachineBasicBlock * getMBB() const
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
MachineOperand * findRegisterDefOperand(unsigned Reg, bool isDead=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
bool addRegisterDead(unsigned Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound=false)
We have determined MI defined a register without a use.
void set(unsigned Idx)
void push_back(const T &Elt)
Definition: SmallVector.h:211
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:382
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
MachineOperand * findRegisterUseOperand(unsigned Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
unsigned Reg
static cl::opt< bool > HardenInterprocedurally(PASS_KEY "-ip", cl::desc("Harden interprocedurally by passing our state in and out of " "functions in the high bits of the stack pointer."), cl::init(true), cl::Hidden)
static const TargetRegisterClass * getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII, unsigned Opcode)
Compute the register class for the unfolded load.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:320
unsigned second
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1185
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
void setIsDead(bool Val=true)
static cl::opt< bool > FenceCallAndRet(PASS_KEY "-fence-call-and-ret", cl::desc("Use a full speculation fence to harden both call and ret edges " "rather than a lighter weight mitigation."), cl::init(false), cl::Hidden)
static bool hasVulnerableLoad(MachineFunction &MF)
Helper to scan a function for loads vulnerable to misspeculation that we want to harden.
void dump() const
dump - Print the current MachineFunction to cerr, useful for debugger use.
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
iterator_range< succ_iterator > successors()
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:343
bool test(unsigned Idx) const
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
CondCode getCondFromBranchOpc(unsigned Opc)
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:648
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
static MachineBasicBlock & splitEdge(MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount, MachineInstr *Br, MachineInstr *&UncondBr, const X86InstrInfo &TII)
static cl::opt< bool > HardenEdgesWithLFENCE(PASS_KEY "-lfence", cl::desc("Use LFENCE along each conditional edge to harden against speculative " "loads rather than conditional movs and poisoned pointers."), cl::init(false), cl::Hidden)
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:266
static cl::opt< bool > EnablePostLoadHardening(PASS_KEY "-post-load", cl::desc("Harden the value loaded *after* it is loaded by " "flushing the loaded bits to 1. This is hard to do " "in general but can be done easily for GPRs."), cl::init(true), cl::Hidden)
static cl::opt< bool > EnableSpeculativeLoadHardening("x86-speculative-load-hardening", cl::desc("Force enable speculative load hardening"), cl::init(false), cl::Hidden)
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
void clear()
Definition: SmallSet.h:218
static void canonicalizePHIOperands(MachineFunction &MF)
Removing duplicate PHI operands to leave the PHI in a canonical and predictable form.
static cl::opt< bool > HardenLoads(PASS_KEY "-loads", cl::desc("Sanitize loads from memory. When disable, no " "significant security is provided."), cl::init(true), cl::Hidden)
void dump() const
Definition: Pass.cpp:129
Memory SSA
Definition: MemorySSA.cpp:64
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol)
Set a symbol that will be emitted just after the instruction itself.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void Initialize(unsigned V)
Initialize - Reset this object to get ready for a new set of SSA updates.
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
Definition: MachineInstr.h:656
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool isReturn(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:622
MCContext & getContext() const
#define PASS_KEY
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:422
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
MCSymbol * createTempSymbol(bool CanBeUnnamed=true)
Create and return a new assembler temporary symbol with a unique but unspecified name.
Definition: MCContext.cpp:216
unsigned const MachineRegisterInfo * MRI
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
bool hasSuperClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a super-class of or equal to this class.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineInstrBuilder & UseMI
LLVM_NODISCARD bool empty() const
Definition: SmallPtrSet.h:91
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
void setMBB(MachineBasicBlock *MBB)
Represent the analysis usage information of a pass.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1192
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:381
unsigned getOperandBias(const MCInstrDesc &Desc)
getOperandBias - compute whether all of the def operands are repeated in the uses and therefore shoul...
Definition: X86BaseInfo.h:655
self_iterator getIterator()
Definition: ilist_node.h:81
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
iterator_range< pred_iterator > predecessors()
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setIsKill(bool Val=true)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1115
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:297
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:377
void initializeX86SpeculativeLoadHardeningPassPass(PassRegistry &)
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
static bool isDataInvariant(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:373
const X86RegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
Definition: X86InstrInfo.h:186
FunctionPass * createX86SpeculativeLoadHardeningPass()
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
const Function & getFunction() const
Return the LLVM function that this machine code represents.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2&#39;s erase_if which is equivalent t...
Definition: STLExtras.h:1329
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
CodeModel::Model getCodeModel() const
Returns the code model.
static cl::opt< bool > HardenIndirectCallsAndJumps(PASS_KEY "-indirect", cl::desc("Harden indirect calls and jumps against using speculatively " "stored attacker controlled addresses. This is designed to " "mitigate Spectre v1.2 style attacks."), cl::init(true), cl::Hidden)
unsigned getCMovFromCond(CondCode CC, unsigned RegBytes, bool HasMemoryOperand=false)
Return a cmov opcode for the given condition, register size in bytes, and operand type...
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:63
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const TargetRegisterInfo &TRI)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
bool isEHPad() const
Returns true if the block is a landing pad.
bool verify(Pass *p=nullptr, const char *Banner=nullptr, bool AbortOnError=true) const
Run the current MachineFunction through the machine code verifier, useful for debugger use...
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
bool exposesReturnsTwice() const
exposesReturnsTwice - Returns true if the function calls setjmp or any other similar functions with a...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
unsigned GetValueInMiddleOfBlock(MachineBasicBlock *BB)
GetValueInMiddleOfBlock - Construct SSA form, materializing a value that is live in the middle of the...
void AddAvailableValue(MachineBasicBlock *BB, unsigned V)
AddAvailableValue - Indicate that a rewritten value is available at the end of the specified block wi...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:171
void splitSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New, bool NormalizeSuccProbs=false)
Split the old successor into old plus new and updates the probability info.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
X86 speculative load hardener
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition: MachineInstr.h:806
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY, "X86 speculative load hardener", false, false) INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass
static bool isDataInvariantLoad(MachineInstr &MI)
Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...
iterator SkipPHIsLabelsAndDebug(iterator I)
Return the first instruction in MBB after I that is not a PHI, label or debug.
unsigned getReg(unsigned Idx)
Get the register for the operand index.
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
unsigned getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore, unsigned *LoadRegIndex=nullptr) const override
getOpcodeAfterMemoryUnfold - Returns the opcode of the would be new instruction after load / store ar...
bool isImplicit() const
int getMemoryOperandNo(uint64_t TSFlags)
getMemoryOperandNo - The function returns the MCInst operand # for the first field of the memory oper...
Definition: X86BaseInfo.h:696
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164