arm64: Force SSBS on context switch

[ Upstream commit cbdf8a189a66001c36007bf0f5c975d0376c5c3a ]

On a CPU that doesn't support SSBS, PSTATE[12] is RES0.  In a system
where only some of the CPUs implement SSBS, we end-up losing track of
the SSBS bit across task migration.

To address this issue, let's force the SSBS bit on context switch.

Fixes: 8f04e8e6e29c ("arm64: ssbd: Add support for PSTATE.SSBS rather than trapping to EL3")
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
[will: inverted logic and added comments]
Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index ad208bd402f7923a985da8a7de2cbfbc0068aa2a..773ea8e0e442136269f9633c6abe0fec47ed1978 100644 (file)
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -177,6 +177,16 @@ static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
        regs->pc = pc;
 }
 
+static inline void set_ssbs_bit(struct pt_regs *regs)
+{
+       regs->pstate |= PSR_SSBS_BIT;
+}
+
+static inline void set_compat_ssbs_bit(struct pt_regs *regs)
+{
+       regs->pstate |= PSR_AA32_SSBS_BIT;
+}
+
 static inline void start_thread(struct pt_regs *regs, unsigned long pc,
                                unsigned long sp)
 {
@@ -184,7 +194,7 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc,
        regs->pstate = PSR_MODE_EL0t;
 
        if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)
-               regs->pstate |= PSR_SSBS_BIT;
+               set_ssbs_bit(regs);
 
        regs->sp = sp;
 }
@@ -203,7 +213,7 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
 #endif
 
        if (arm64_get_ssbd_state() != ARM64_SSBD_FORCE_ENABLE)
-               regs->pstate |= PSR_AA32_SSBS_BIT;
+               set_compat_ssbs_bit(regs);
 
        regs->compat_sp = sp;
 }

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index ce99c58cd1f1d2081355a7f4420072a31b43ca71..bc2226608e13264920d57057da6826ef8e313e43 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -360,7 +360,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
                        childregs->pstate |= PSR_UAO_BIT;
 
                if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)
-                       childregs->pstate |= PSR_SSBS_BIT;
+                       set_ssbs_bit(childregs);
 
                p->thread.cpu_context.x19 = stack_start;
                p->thread.cpu_context.x20 = stk_sz;
@@ -401,6 +401,32 @@ void uao_thread_switch(struct task_struct *next)
        }
 }
 
+/*
+ * Force SSBS state on context-switch, since it may be lost after migrating
+ * from a CPU which treats the bit as RES0 in a heterogeneous system.
+ */
+static void ssbs_thread_switch(struct task_struct *next)
+{
+       struct pt_regs *regs = task_pt_regs(next);
+
+       /*
+        * Nothing to do for kernel threads, but 'regs' may be junk
+        * (e.g. idle task) so check the flags and bail early.
+        */
+       if (unlikely(next->flags & PF_KTHREAD))
+               return;
+
+       /* If the mitigation is enabled, then we leave SSBS clear. */
+       if ((arm64_get_ssbd_state() == ARM64_SSBD_FORCE_ENABLE) ||
+           test_tsk_thread_flag(next, TIF_SSBD))
+               return;
+
+       if (compat_user_mode(regs))
+               set_compat_ssbs_bit(regs);
+       else if (user_mode(regs))
+               set_ssbs_bit(regs);
+}
+
 /*
  * We store our current task in sp_el0, which is clobbered by userspace. Keep a
  * shadow copy so that we can restore this upon entry from userspace.
@@ -429,6 +455,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
        contextidr_thread_switch(next);
        entry_task_switch(next);
        uao_thread_switch(next);
+       ssbs_thread_switch(next);
 
        /*
         * Complete any pending TLB or cache maintenance on this CPU in case