x86/speculation: Add eIBRS + Retpoline options

commit 1e19da8522c81bf46b335f84137165741e0d82b7 upstream.

Thanks to the chaps at VUsec it is now clear that eIBRS is not
sufficient, therefore allow enabling of retpolines along with eIBRS.

Add spectre_v2=eibrs, spectre_v2=eibrs,lfence and
spectre_v2=eibrs,retpoline options to explicitly pick your preferred
means of mitigation.

Since there's new mitigations there's also user visible changes in
/sys/devices/system/cpu/vulnerabilities/spectre_v2 to reflect these
new mitigations.

  [ bp: Massage commit message, trim error messages,
    do more precise eIBRS mode checking. ]

Co-developed-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Patrick Colp <patrick.colp@oracle.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
[fllinden@amazon.com: backported to 4.19 (no Hygon)]
Signed-off-by: Frank van der Linden <fllinden@amazon.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 5baa5c7c0419814d6ee8219d6e75a8fc8efe1975..812976a6d864207d63d6f59c2257c18b7ab166db 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -225,7 +225,9 @@ enum spectre_v2_mitigation {
        SPECTRE_V2_NONE,
        SPECTRE_V2_RETPOLINE,
        SPECTRE_V2_LFENCE,
-       SPECTRE_V2_IBRS_ENHANCED,
+       SPECTRE_V2_EIBRS,
+       SPECTRE_V2_EIBRS_RETPOLINE,
+       SPECTRE_V2_EIBRS_LFENCE,
 };
 
 /* The indirect branch speculation control variants */

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 494d85f5922276461fc10e6a21a0296a0f2f42e8..09dd0d596f9c46413f7ea8f0047d51db4ed756b8 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -622,6 +622,9 @@ enum spectre_v2_mitigation_cmd {
        SPECTRE_V2_CMD_RETPOLINE,
        SPECTRE_V2_CMD_RETPOLINE_GENERIC,
        SPECTRE_V2_CMD_RETPOLINE_LFENCE,
+       SPECTRE_V2_CMD_EIBRS,
+       SPECTRE_V2_CMD_EIBRS_RETPOLINE,
+       SPECTRE_V2_CMD_EIBRS_LFENCE,
 };
 
 enum spectre_v2_user_cmd {
@@ -694,6 +697,13 @@ spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
        return SPECTRE_V2_USER_CMD_AUTO;
 }
 
+static inline bool spectre_v2_in_eibrs_mode(enum spectre_v2_mitigation mode)
+{
+       return (mode == SPECTRE_V2_EIBRS ||
+               mode == SPECTRE_V2_EIBRS_RETPOLINE ||
+               mode == SPECTRE_V2_EIBRS_LFENCE);
+}
+
 static void __init
 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 {
@@ -761,7 +771,7 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
         */
        if (!boot_cpu_has(X86_FEATURE_STIBP) ||
            !smt_possible ||
-           spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
+           spectre_v2_in_eibrs_mode(spectre_v2_enabled))
                return;
 
        /*
@@ -783,7 +793,9 @@ static const char * const spectre_v2_strings[] = {
        [SPECTRE_V2_NONE]                       = "Vulnerable",
        [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
        [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
-       [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
+       [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced IBRS",
+       [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced IBRS + LFENCE",
+       [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced IBRS + Retpolines",
 };
 
 static const struct {
@@ -797,6 +809,9 @@ static const struct {
        { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
        { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
        { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
+       { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
+       { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
+       { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
        { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 };
 
@@ -834,15 +849,29 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 
        if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
             cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
-            cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
+            cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
+            cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
+            cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
            !IS_ENABLED(CONFIG_RETPOLINE)) {
-               pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
+               pr_err("%s selected but not compiled in. Switching to AUTO select\n",
+                      mitigation_options[i].option);
+               return SPECTRE_V2_CMD_AUTO;
+       }
+
+       if ((cmd == SPECTRE_V2_CMD_EIBRS ||
+            cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
+            cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
+           !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
+               pr_err("%s selected but CPU doesn't have eIBRS. Switching to AUTO select\n",
+                      mitigation_options[i].option);
                return SPECTRE_V2_CMD_AUTO;
        }
 
-       if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE) &&
+       if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
+            cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
            !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
-               pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n", mitigation_options[i].option);
+               pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
+                      mitigation_options[i].option);
                return SPECTRE_V2_CMD_AUTO;
        }
 
@@ -851,6 +880,24 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
        return cmd;
 }
 
+static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
+{
+       if (!IS_ENABLED(CONFIG_RETPOLINE)) {
+               pr_err("Kernel not compiled with retpoline; no mitigation available!");
+               return SPECTRE_V2_NONE;
+       }
+
+       if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
+               if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
+                       pr_err("LFENCE not serializing, switching to generic retpoline\n");
+                       return SPECTRE_V2_RETPOLINE;
+               }
+               return SPECTRE_V2_LFENCE;
+       }
+
+       return SPECTRE_V2_RETPOLINE;
+}
+
 static void __init spectre_v2_select_mitigation(void)
 {
        enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -871,48 +918,60 @@ static void __init spectre_v2_select_mitigation(void)
        case SPECTRE_V2_CMD_FORCE:
        case SPECTRE_V2_CMD_AUTO:
                if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
-                       mode = SPECTRE_V2_IBRS_ENHANCED;
-                       /* Force it so VMEXIT will restore correctly */
-                       x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
-                       wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
-                       goto specv2_set_mode;
+                       mode = SPECTRE_V2_EIBRS;
+                       break;
                }
-               if (IS_ENABLED(CONFIG_RETPOLINE))
-                       goto retpoline_auto;
+
+               mode = spectre_v2_select_retpoline();
                break;
+
        case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
-               if (IS_ENABLED(CONFIG_RETPOLINE))
-                       goto retpoline_lfence;
+               mode = SPECTRE_V2_LFENCE;
                break;
+
        case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
-               if (IS_ENABLED(CONFIG_RETPOLINE))
-                       goto retpoline_generic;
+               mode = SPECTRE_V2_RETPOLINE;
                break;
+
        case SPECTRE_V2_CMD_RETPOLINE:
-               if (IS_ENABLED(CONFIG_RETPOLINE))
-                       goto retpoline_auto;
+               mode = spectre_v2_select_retpoline();
+               break;
+
+       case SPECTRE_V2_CMD_EIBRS:
+               mode = SPECTRE_V2_EIBRS;
+               break;
+
+       case SPECTRE_V2_CMD_EIBRS_LFENCE:
+               mode = SPECTRE_V2_EIBRS_LFENCE;
+               break;
+
+       case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
+               mode = SPECTRE_V2_EIBRS_RETPOLINE;
                break;
        }
-       pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
-       return;
 
-retpoline_auto:
-       if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
-       retpoline_lfence:
-               if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
-                       pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
-                       goto retpoline_generic;
-               }
-               mode = SPECTRE_V2_LFENCE;
+       if (spectre_v2_in_eibrs_mode(mode)) {
+               /* Force it so VMEXIT will restore correctly */
+               x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
+               wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
+       }
+
+       switch (mode) {
+       case SPECTRE_V2_NONE:
+       case SPECTRE_V2_EIBRS:
+               break;
+
+       case SPECTRE_V2_LFENCE:
+       case SPECTRE_V2_EIBRS_LFENCE:
                setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
+               /* fallthrough */
+
+       case SPECTRE_V2_RETPOLINE:
+       case SPECTRE_V2_EIBRS_RETPOLINE:
                setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
-       } else {
-       retpoline_generic:
-               mode = SPECTRE_V2_RETPOLINE;
-               setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
+               break;
        }
 
-specv2_set_mode:
        spectre_v2_enabled = mode;
        pr_info("%s\n", spectre_v2_strings[mode]);
 
@@ -938,7 +997,7 @@ specv2_set_mode:
         * the CPU supports Enhanced IBRS, kernel might un-intentionally not
         * enable IBRS around firmware calls.
         */
-       if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
+       if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_eibrs_mode(mode)) {
                setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
                pr_info("Enabling Restricted Speculation for firmware calls\n");
        }
@@ -1596,7 +1655,7 @@ static ssize_t tsx_async_abort_show_state(char *buf)
 
 static char *stibp_state(void)
 {
-       if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
+       if (spectre_v2_in_eibrs_mode(spectre_v2_enabled))
                return "";
 
        switch (spectre_v2_user_stibp) {