KVM: arm64: nvhe: Save the SPE context early

commit b96b0c5de685df82019e16826a282d53d86d112c upstream

The nVHE KVM hyp drains and disables the SPE buffer, before
entering the guest, as the EL1&0 translation regime
is going to be loaded with that of the guest.

But this operation is performed way too late, because :
 - The owning translation regime of the SPE buffer
   is transferred to EL2. (MDCR_EL2_E2PB == 0)
 - The guest Stage1 is loaded.

Thus the flush could use the host EL1 virtual address,
but use the EL2 translations instead of host EL1, for writing
out any cached data.

Fix this by moving the SPE buffer handling early enough.
The restore path is doing the right thing.

Cc: stable@vger.kernel.org # v5.4-
Cc: Christoffer Dall <christoffer.dall@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Alexandru Elisei <alexandru.elisei@arm.com>
Signed-off-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Acked-by: Marc Zyngier <maz@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 97f21cc666579c350ca625f26e4e84012bd77096..7f7fdb16bb9680a64945e62969f5fed8dd0425a5 100644 (file)
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -71,6 +71,9 @@ void __sysreg32_restore_state(struct kvm_vcpu *vcpu);
 
 void __debug_switch_to_guest(struct kvm_vcpu *vcpu);
 void __debug_switch_to_host(struct kvm_vcpu *vcpu);
+void __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu);
+void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu);
+
 
 void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
 void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);

diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c
index 0fc9872a1467107203dfca22867c6887f3eff551..aead8a5fbe91940f5344ba0dc62d70aab8e84913 100644 (file)
--- a/arch/arm64/kvm/hyp/debug-sr.c
+++ b/arch/arm64/kvm/hyp/debug-sr.c
@@ -168,6 +168,21 @@ static void __hyp_text __debug_restore_state(struct kvm_vcpu *vcpu,
        write_sysreg(ctxt->sys_regs[MDCCINT_EL1], mdccint_el1);
 }
 
+void __hyp_text __debug_save_host_buffers_nvhe(struct kvm_vcpu *vcpu)
+{
+       /*
+        * Non-VHE: Disable and flush SPE data generation
+        * VHE: The vcpu can run, but it can't hide.
+        */
+       __debug_save_spe_nvhe(&vcpu->arch.host_debug_state.pmscr_el1);
+
+}
+
+void __hyp_text __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu)
+{
+       __debug_restore_spe_nvhe(vcpu->arch.host_debug_state.pmscr_el1);
+}
+
 void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu)
 {
        struct kvm_cpu_context *host_ctxt;
@@ -175,13 +190,6 @@ void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu)
        struct kvm_guest_debug_arch *host_dbg;
        struct kvm_guest_debug_arch *guest_dbg;
 
-       /*
-        * Non-VHE: Disable and flush SPE data generation
-        * VHE: The vcpu can run, but it can't hide.
-        */
-       if (!has_vhe())
-               __debug_save_spe_nvhe(&vcpu->arch.host_debug_state.pmscr_el1);
-
        if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY))
                return;
 
@@ -201,8 +209,6 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu)
        struct kvm_guest_debug_arch *host_dbg;
        struct kvm_guest_debug_arch *guest_dbg;
 
-       if (!has_vhe())
-               __debug_restore_spe_nvhe(vcpu->arch.host_debug_state.pmscr_el1);
 
        if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY))
                return;

diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index 84964983198e2214fe919ca3fa86b2339fcb28f5..14607fac7ca38364f88ea052545a475366f755d8 100644 (file)
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -682,6 +682,15 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 
        __sysreg_save_state_nvhe(host_ctxt);
 
+       /*
+        * We must flush and disable the SPE buffer for nVHE, as
+        * the translation regime(EL1&0) is going to be loaded with
+        * that of the guest. And we must do this before we change the
+        * translation regime to EL2 (via MDCR_EL2_EPB == 0) and
+        * before we load guest Stage1.
+        */
+       __debug_save_host_buffers_nvhe(vcpu);
+
        __activate_vm(kern_hyp_va(vcpu->kvm));
        __activate_traps(vcpu);
 
@@ -720,11 +729,13 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
        if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
                __fpsimd_save_fpexc32(vcpu);
 
+       __debug_switch_to_host(vcpu);
+
        /*
         * This must come after restoring the host sysregs, since a non-VHE
         * system may enable SPE here and make use of the TTBRs.
         */
-       __debug_switch_to_host(vcpu);
+       __debug_restore_host_buffers_nvhe(vcpu);
 
        if (pmu_switch_needed)
                __pmu_switch_to_host(host_ctxt);