File 5a6b36cd-6-x86-clobber-RSB-RAS-on-entry.patch of Package xen.8005

# Commit e6c0128e9ab25bf66df11377a33ee5584d7f99e3
# Date 2018-01-26 14:10:21 +0000
# Author Andrew Cooper <andrew.cooper3@citrix.com>
# Committer Andrew Cooper <andrew.cooper3@citrix.com>
x86/entry: Organise the clobbering of the RSB/RAS on entry to Xen

ret instructions are speculated directly to values recorded in the Return
Stack Buffer/Return Address Stack, as there is no uncertainty in well-formed
code.  Guests can take advantage of this in two ways:

  1) If they can find a path in Xen which executes more ret instructions than
     call instructions.  (At least one in the waitqueue infrastructure,
     probably others.)

  2) Use the fact that the RSB/RAS in hardware is actually a circular stack
     without a concept of empty.  (When it logically empties, stale values
     will start being used.)

To mitigate, overwrite the RSB on entry to Xen with gadgets which will capture
and contain rogue speculation.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

--- a/xen/arch/x86/x86_64/asm-offsets.c
+++ b/xen/arch/x86/x86_64/asm-offsets.c
@@ -140,6 +140,7 @@ void __dummy__(void)
     OFFSET(CPUINFO_shadow_spec_ctrl, struct cpu_info, shadow_spec_ctrl);
     OFFSET(CPUINFO_use_shadow_spec_ctrl, struct cpu_info, use_shadow_spec_ctrl);
     OFFSET(CPUINFO_xen_ibrs, struct cpu_info, xen_ibrs);
+    OFFSET(CPUINFO_xen_rsb, struct cpu_info, xen_rsb);
     DEFINE(CPUINFO_sizeof, sizeof(struct cpu_info));
     BLANK();
 
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -12,6 +12,11 @@
 #include <public/xen.h>
 #include <asm/page.h>
 
+#define XEN_RSB_VMEXIT 0x01
+#define XEN_RSB_NATIVE 0x02
+
+#ifndef __ASSEMBLY__
+
 struct vcpu;
 
 struct cpu_info {
@@ -36,6 +41,7 @@ struct cpu_info {
     unsigned int shadow_spec_ctrl;
     bool_t       use_shadow_spec_ctrl;
     int8_t       xen_ibrs;
+    uint8_t      xen_rsb;
 
     /* get_stack_bottom() must be 16-byte aligned */
 };
@@ -90,4 +96,6 @@ static inline struct cpu_info *get_cpu_i
  */
 DECLARE_PER_CPU(struct vcpu *, curr_vcpu);
 
+#endif /* __ASSEMBLY__ */
+
 #endif /* __X86_CURRENT_H__ */
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -30,6 +30,7 @@ static inline void init_shadow_spec_ctrl
 
     info->shadow_spec_ctrl = info->use_shadow_spec_ctrl = 0;
     info->xen_ibrs = -1;
+    info->xen_rsb = 0;
 }
 
 #endif /* !__X86_SPEC_CTRL_H__ */
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -21,6 +21,7 @@
 #define __X86_SPEC_CTRL_ASM_H__
 
 #ifdef __ASSEMBLY__
+#include <asm/current.h>
 #include <asm/msr-index.h>
 
 /*
@@ -74,6 +75,53 @@
  *  - SPEC_CTRL_EXIT_TO_GUEST
  */
 
+.macro DO_OVERWRITE_RSB mask:req maybexen=0
+/*
+ * Requires %rsp=cpuinfo (if !maybexen)
+ * Requires %r14=stack_end (if maybexen)
+ * Clobbers %rax, %rcx
+ *
+ * Requires 256 bytes of stack space, but %rsp has no net change. Based on
+ * Google's performance numbers, the loop is unrolled to 16 iterations and two
+ * calls per iteration.
+ *
+ * The call filling the RSB needs a nonzero displacement.  A nop would do, but
+ * we use "1: pause; lfence; jmp 1b" to safely contains any ret-based
+ * speculation, even if the loop is speculatively executed prematurely.
+ *
+ * %rsp is preserved by using an extra GPR because a) we've got plenty spare,
+ * b) the two movs are shorter to encode than `add $32*8, %rsp`, and c) can be
+ * optimised with mov-elimination in modern cores.
+ */
+    .if \maybexen
+        testb $\mask, STACK_CPUINFO_FIELD(xen_rsb)(%r14)
+    .else
+        testb $\mask, CPUINFO_xen_rsb(%rsp)
+    .endif
+    jz .Lrsb_done\@
+
+    mov $16, %ecx                   /* 16 iterations, two calls per loop */
+    mov %rsp, %rax                  /* Store the current %rsp */
+
+.L\@_fill_rsb_loop:
+
+    .irp n, 1, 2                    /* Unrolled twice. */
+    call .L\@_insert_rsb_entry_\n   /* Create an RSB entry. */
+
+.L\@_capture_speculation_\n:
+    pause
+    lfence
+    jmp .L\@_capture_speculation_\n /* Capture rogue speculation. */
+
+.L\@_insert_rsb_entry_\n:
+    .endr
+
+    sub $1, %ecx
+    jnz .L\@_fill_rsb_loop
+    mov %rax, %rsp                  /* Restore old %rsp */
+.Lrsb_done\@:
+.endm
+
 .macro DO_SPEC_CTRL_ENTRY_FROM_VMEXIT
 /*
  * Requires %rbx=current, %rsp=regs/cpuinfo
@@ -193,14 +241,17 @@
 
 /* Use after a VMEXIT from an HVM guest. */
 #define SPEC_CTRL_ENTRY_FROM_VMEXIT                                     \
+        DO_OVERWRITE_RSB XEN_RSB_VMEXIT;                                \
         DO_SPEC_CTRL_ENTRY_FROM_VMEXIT
 
 /* Use after an entry from PV context (syscall/sysenter/int80/int82/etc). */
 #define SPEC_CTRL_ENTRY_FROM_PV                                         \
+        DO_OVERWRITE_RSB XEN_RSB_NATIVE;                                \
         DO_SPEC_CTRL_ENTRY maybexen=0
 
 /* Use in interrupt/exception context.  May interrupt Xen or PV context. */
 #define SPEC_CTRL_ENTRY_FROM_INTR                                       \
+        DO_OVERWRITE_RSB mask=XEN_RSB_NATIVE maybexen=1;                \
         DO_SPEC_CTRL_ENTRY maybexen=1
 
 /* Use when exiting to Xen context. */