arm/arm64: KVM: Perform local TLB invalidation when multiplexing vcpus on a single CPU

Architecturally, TLBs are private to the (physical) CPU they're
associated with. But when multiple vcpus from the same VM are
being multiplexed on the same CPU, the TLBs are not private
to the vcpus (and are actually shared across the VMID).

Let's consider the following scenario:

- vcpu-0 maps PA to VA
- vcpu-1 maps PA' to VA

If run on the same physical CPU, vcpu-1 can hit TLB entries generated
by vcpu-0 accesses, and access the wrong physical page.

The solution to this is to keep a per-VM map of which vcpu ran last
on each given physical CPU, and invalidate local TLBs when switching
to a different vcpu from the same VM.

Reviewed-by: Christoffer Dall <[email protected]>
Signed-off-by: Marc Zyngier <[email protected]>

diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index d7ea6bcb29bf68489323f1c777e057de0b81e364..8ef05381984b1b6ba977035c82607423b37835c0 100644 (file)
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -66,6 +66,7 @@ extern char __kvm_hyp_vector[];
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
 extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 

diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 2d19e02d03fd69e75f327659f582d8abbdf909e9..d5423ab15ed5be1c705817e13d4a7d7fe35b465b 100644 (file)
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -57,6 +57,9 @@ struct kvm_arch {
        /* VTTBR value associated with below pgd and vmid */
        u64    vttbr;
 
+       /* The last vcpu id that ran on each physical CPU */
+       int __percpu *last_vcpu_ran;
+
        /* Timer */
        struct arch_timer_kvm   timer;
 

diff --git a/arch/arm/include/asm/kvm_hyp.h b/arch/arm/include/asm/kvm_hyp.h
index 343135ede5fa01d848ec609f0eafc08f610f5aab..58508900c4bb264be2a874299b829b31a9cb5601 100644 (file)
--- a/arch/arm/include/asm/kvm_hyp.h
+++ b/arch/arm/include/asm/kvm_hyp.h
@@ -71,6 +71,7 @@
 #define ICIALLUIS      __ACCESS_CP15(c7, 0, c1, 0)
 #define ATS1CPR                __ACCESS_CP15(c7, 0, c8, 0)
 #define TLBIALLIS      __ACCESS_CP15(c8, 0, c3, 0)
+#define TLBIALL                __ACCESS_CP15(c8, 0, c7, 0)
 #define TLBIALLNSNHIS  __ACCESS_CP15(c8, 4, c3, 4)
 #define PRRR           __ACCESS_CP15(c10, 0, c2, 0)
 #define NMRR           __ACCESS_CP15(c10, 0, c2, 1)

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 08bb84f2ad58e6be8caf539f6d8189c1b450cfa1..19b5f5c1c0ff3ef8fa68300f5ec87fe5564c916e 100644 (file)
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -114,11 +114,18 @@ void kvm_arch_check_processor_compat(void *rtn)
  */
 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 {
-       int ret = 0;
+       int ret, cpu;
 
        if (type)
                return -EINVAL;
 
+       kvm->arch.last_vcpu_ran = alloc_percpu(typeof(*kvm->arch.last_vcpu_ran));
+       if (!kvm->arch.last_vcpu_ran)
+               return -ENOMEM;
+
+       for_each_possible_cpu(cpu)
+               *per_cpu_ptr(kvm->arch.last_vcpu_ran, cpu) = -1;
+
        ret = kvm_alloc_stage2_pgd(kvm);
        if (ret)
                goto out_fail_alloc;
@@ -141,6 +148,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 out_free_stage2_pgd:
        kvm_free_stage2_pgd(kvm);
 out_fail_alloc:
+       free_percpu(kvm->arch.last_vcpu_ran);
+       kvm->arch.last_vcpu_ran = NULL;
        return ret;
 }
 
@@ -168,6 +177,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 {
        int i;
 
+       free_percpu(kvm->arch.last_vcpu_ran);
+       kvm->arch.last_vcpu_ran = NULL;
+
        for (i = 0; i < KVM_MAX_VCPUS; ++i) {
                if (kvm->vcpus[i]) {
                        kvm_arch_vcpu_free(kvm->vcpus[i]);
@@ -312,6 +324,19 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
+       int *last_ran;
+
+       last_ran = this_cpu_ptr(vcpu->kvm->arch.last_vcpu_ran);
+
+       /*
+        * We might get preempted before the vCPU actually runs, but
+        * over-invalidation doesn't affect correctness.
+        */
+       if (*last_ran != vcpu->vcpu_id) {
+               kvm_call_hyp(__kvm_tlb_flush_local_vmid, vcpu);
+               *last_ran = vcpu->vcpu_id;
+       }
+
        vcpu->cpu = cpu;
        vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
 

diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 729652854f9098d677bd59871452c7b8c1ea240b..6d810af2d9fd7c630603ee5bfa8108c42a9992d8 100644 (file)
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -55,6 +55,21 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
        __kvm_tlb_flush_vmid(kvm);
 }
 
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+{
+       struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
+
+       /* Switch to requested VMID */
+       write_sysreg(kvm->arch.vttbr, VTTBR);
+       isb();
+
+       write_sysreg(0, TLBIALL);
+       dsb(nsh);
+       isb();
+
+       write_sysreg(0, VTTBR);
+}
+
 void __hyp_text __kvm_flush_vm_context(void)
 {
        write_sysreg(0, TLBIALLNSNHIS);

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 18f746551bf632cc88a3a406359463e9d49340eb..ec3553eb9349093a9c7675f7fbe4de38dba396c7 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -54,6 +54,7 @@ extern char __kvm_hyp_vector[];
 extern void __kvm_flush_vm_context(void);
 extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
 extern void __kvm_tlb_flush_vmid(struct kvm *kvm);
+extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index bd94e67667599dc1ce499ab30de2cca91f4e97be..e5050388e062209868bac64cab1740ece15b3e13 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -62,6 +62,9 @@ struct kvm_arch {
        /* VTTBR value associated with above pgd and vmid */
        u64    vttbr;
 
+       /* The last vcpu id that ran on each physical CPU */
+       int __percpu *last_vcpu_ran;
+
        /* The maximum number of vCPUs depends on the used GIC model */
        int max_vcpus;
 

diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index a79b969c26fca7dcd73e06c4bfc36c6519cb646b..6f72fe8b0e3ee477a076fa5f9ef02d13c3d8eef3 100644 (file)
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -128,7 +128,7 @@ static inline unsigned long __kern_hyp_va(unsigned long v)
        return v;
 }
 
-#define kern_hyp_va(v)         (typeof(v))(__kern_hyp_va((unsigned long)(v)))
+#define kern_hyp_va(v)         ((typeof(v))(__kern_hyp_va((unsigned long)(v))))
 
 /*
  * We currently only support a 40bit IPA.

diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 9cc0ea784ae60a0a5086f1c5f6d129cd7d360f13..88e2f2b938f070c7570a8d76ae9ca348b1fd71e9 100644 (file)
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -64,6 +64,21 @@ void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
        write_sysreg(0, vttbr_el2);
 }
 
+void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
+{
+       struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
+
+       /* Switch to requested VMID */
+       write_sysreg(kvm->arch.vttbr, vttbr_el2);
+       isb();
+
+       asm volatile("tlbi vmalle1" : : );
+       dsb(nsh);
+       isb();
+
+       write_sysreg(0, vttbr_el2);
+}
+
 void __hyp_text __kvm_flush_vm_context(void)
 {
        dsb(ishst);