File xsa422-02.patch of Package xen.28177
Subject: x86/spec-ctrl: Mitigate IBPB not flushing the RSB/RAS
From: Andrew Cooper andrew.cooper3@citrix.com Tue Jun 14 16:18:36 2022 +0100
Date: Fri Nov 4 13:24:37 2022 +0000:
Git: 1151d260d7a0186978b80b708fcb712eb1470f49
Introduce spec_ctrl_new_guest_context() to encapsulate all logic pertaining to
using MSR_PRED_CMD for a new guest context, even if it only has one user
presently.
Introduce X86_BUG_IBPB_NO_RET, and use it extend spec_ctrl_new_guest_context()
with a manual fixup for hardware which mis-implements IBPB.
This is part of XSA-422 / CVE-2022-23824.
Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Jan Beulich <jbeulich@suse.com>
(cherry picked from commit 2b27967fb89d7904a1571a2fb963b1c9cac548db)
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -2314,7 +2314,7 @@ void context_switch(struct vcpu *prev, s
*/
if ( *last_id != next_id )
{
- wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
+ spec_ctrl_new_guest_context();
*last_id = next_id;
}
}
--- a/xen/arch/x86/spec_ctrl.c
+++ b/xen/arch/x86/spec_ctrl.c
@@ -784,6 +784,14 @@ static void __init ibpb_calculations(voi
}
/*
+ * AMD/Hygon CPUs to date (June 2022) don't flush the the RAS. Future
+ * CPUs are expected to enumerate IBPB_RET when this has been fixed.
+ * Until then, cover the difference with the software sequence.
+ */
+ if ( boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_IBPB_RET) )
+ __set_bit(X86_BUG_IBPB_NO_RET, boot_cpu_data.x86_capability);
+
+ /*
* IBPB-on-entry mitigations for Branch Type Confusion.
*
* IBPB && !BTC_NO selects all AMD/Hygon hardware, not known to be safe,
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -43,6 +43,7 @@ XEN_CPUFEATURE(IBPB_ENTRY_HVM, (FSCAPIN
#define X86_BUG(x) ((FSCAPINTS + X86_NR_SYNTH) * 32 + (x))
#define X86_BUG_CLFLUSH_MFENCE X86_BUG( 2) /* MFENCE needed to serialise CLFLUSH */
+#define X86_BUG_IBPB_NO_RET X86_BUG( 3) /* IBPB doesn't flush the RSB/RAS */
/* Total number of capability words, inc synth and bug words. */
#define NCAPINTS (FSCAPINTS + X86_NR_SYNTH + X86_NR_BUG) /* N 32-bit words worth of info */
--- a/xen/include/asm-x86/nops.h
+++ b/xen/include/asm-x86/nops.h
@@ -51,7 +51,7 @@
#ifdef __ASSEMBLY__
#define _ASM_MK_NOP(x) .byte x
#else
-#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n"
+#define _ASM_MK_NOP(x...) ".byte " __stringify(x) "\n"
#endif
#define ASM_NOP1 _ASM_MK_NOP(K8_NOP1)
@@ -63,6 +63,7 @@
#define ASM_NOP7 _ASM_MK_NOP(K8_NOP7)
#define ASM_NOP8 _ASM_MK_NOP(K8_NOP8)
+#ifdef __ASSEMBLY__
#define ASM_NOP17 ASM_NOP8; ASM_NOP7; ASM_NOP2
#define ASM_NOP22 ASM_NOP8; ASM_NOP8; ASM_NOP6
#define ASM_NOP24 ASM_NOP8; ASM_NOP8; ASM_NOP8
@@ -72,6 +73,9 @@
#define ASM_NOP34 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP7; ASM_NOP3
#define ASM_NOP36 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP4
#define ASM_NOP40 ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8; ASM_NOP8
+#else
+#define ASM_NOP40 _ASM_MK_NOP(K8_NOP8, K8_NOP8, K8_NOP8, K8_NOP8, K8_NOP8)
+#endif
#define ASM_NOP_MAX 8
--- a/xen/include/asm-x86/spec_ctrl.h
+++ b/xen/include/asm-x86/spec_ctrl.h
@@ -22,11 +22,33 @@
#include <asm/alternative.h>
#include <asm/current.h>
-#include <asm/msr-index.h>
+#include <asm/msr.h>
void init_speculation_mitigations(void);
void spec_ctrl_init_domain(struct domain *d);
+/*
+ * Switch to a new guest prediction context.
+ *
+ * This flushes all indirect branch predictors (BTB, RSB/RAS), so guest code
+ * which has previously run on this CPU can't attack subsequent guest code.
+ *
+ * As this flushes the RSB/RAS, it destroys the predictions of the calling
+ * context. For best performace, arrange for this to be used when we're going
+ * to jump out of the current context, e.g. with reset_stack_and_jump().
+ *
+ * For hardware which mis-implements IBPB, fix up by flushing the RSB/RAS
+ * manually.
+ */
+static always_inline void spec_ctrl_new_guest_context(void)
+{
+ wrmsrl(MSR_PRED_CMD, PRED_CMD_IBPB);
+
+ /* (ab)use alternative_input() to specify clobbers. */
+ alternative_input(ASM_NOP40, "DO_OVERWRITE_RSB", X86_BUG_IBPB_NO_RET,
+ : "rax", "rcx");
+}
+
extern int8_t opt_ibpb_ctxt_switch;
extern bool_t opt_ssbd;
extern int8_t opt_eager_fpu;
--- a/xen/include/asm-x86/spec_ctrl_asm.h
+++ b/xen/include/asm-x86/spec_ctrl_asm.h
@@ -144,10 +144,22 @@
.L\@_done:
.endm
-.macro DO_OVERWRITE_RSB tmp=rax
+#define LBL(name) .L\@_##name
+#define LBLn(name) .L\@_##name##_\n
+#define asm(insn...) insn
+
+#else /* !__ASSEMBLY__ */
+
+#define LBL(name) .L\\@_##name
+#define LBLn(name) .L\\@_##name##_\\n
+#define asm(insn...) asm ( __stringify(insn) )
+
+#endif /* __ASSEMBLY__ */
+
+asm(.macro DO_OVERWRITE_RSB;
/*
* Requires nothing
- * Clobbers \tmp (%rax by default), %rcx
+ * Clobbers %rax, %rcx
*
* Requires 256 bytes of stack space, but %rsp has no net change. Based on
* Google's performance numbers, the loop is unrolled to 16 iterations and two
@@ -161,26 +173,32 @@
* b) the two movs are shorter to encode than `add $32*8, %rsp`, and c) can be
* optimised with mov-elimination in modern cores.
*/
- mov $16, %ecx /* 16 iterations, two calls per loop */
- mov %rsp, %\tmp /* Store the current %rsp */
+ mov $16, %ecx; /* 16 iterations, two calls per loop */
+ mov %rsp, %rax; /* Store the current %rsp */
-.L\@_fill_rsb_loop:
+LBL(fill_rsb_loop):;
- .irp n, 1, 2 /* Unrolled twice. */
- call .L\@_insert_rsb_entry_\n /* Create an RSB entry. */
+ .irp n, 1, 2; /* Unrolled twice. */
+ call LBLn(insert_rsb_entry); /* Create an RSB entry. */
-.L\@_capture_speculation_\n:
- pause
- lfence
- jmp .L\@_capture_speculation_\n /* Capture rogue speculation. */
+LBLn(capture_speculation):
+ pause;
+ lfence;
+ jmp LBLn(capture_speculation); /* Capture rogue speculation. */
-.L\@_insert_rsb_entry_\n:
- .endr
+LBLn(insert_rsb_entry):;
+ .endr;
- sub $1, %ecx
- jnz .L\@_fill_rsb_loop
- mov %\tmp, %rsp /* Restore old %rsp */
-.endm
+ sub $1, %ecx;
+ jnz LBL(fill_rsb_loop);
+ mov %rax, %rsp; /* Restore old %rsp */
+.endm);
+
+#undef asm
+#undef LBLn
+#undef LBL
+
+#ifdef __ASSEMBLY__
.macro DO_SPEC_CTRL_ENTRY_FROM_HVM
/*
