KVM: arm/arm64: Factor out VMID into struct kvm_vmid

In preparation for nested virtualization where we are going to have more
than a single VMID per VM, let's factor out the VMID data into a
separate VMID data structure and change the VMID allocator to operate on
this new structure instead of using a struct kvm.

This also means that udate_vttbr now becomes update_vmid, and that the
vttbr itself is generated on the fly based on the stage 2 page table
base address and the vmid.

We cache the physical address of the pgd when allocating the pgd to
avoid doing the calculation on every entry to the guest and to avoid
calling into potentially non-hyp-mapped code from hyp/EL2.

If we wanted to merge the VMID allocator with the arm64 ASID allocator
at some point in the future, it should actually become easier to do that
after this patch.

Note that to avoid mapping the kvm_vmid_bits variable into hyp, we
simply forego the masking of the vmid value in kvm_get_vttbr and rely on
update_vmid to always assign a valid vmid value (within the supported
range).

Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
[maz: minor cleanups]
Reviewed-by: Julien Thierry <julien.thierry@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>

2 files changed, 27 insertions, 37 deletions

diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 3dd240ea9e76..b77db673bb03 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -65,7 +65,6 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
 /* The VMID used in the VTTBR */
 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
 static u32 kvm_next_vmid;
-static unsigned int kvm_vmid_bits __read_mostly;
 static DEFINE_SPINLOCK(kvm_vmid_lock);
 
 static bool vgic_present;
@@ -142,7 +141,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
         kvm_vgic_early_init(kvm);
 
         /* Mark the initial VMID generation invalid */
-        kvm->arch.vmid_gen = 0;
+        kvm->arch.vmid.vmid_gen = 0;
 
         /* The maximum number of VCPUs is limited by the host's GIC model */
         kvm->arch.max_vcpus = vgic_present ?
@@ -472,37 +471,31 @@ void force_vm_exit(const cpumask_t *mask)
 
 /**
  * need_new_vmid_gen - check that the VMID is still valid
- * @kvm: The VM's VMID to check
+ * @vmid: The VMID to check
  *
  * return true if there is a new generation of VMIDs being used
  *
- * The hardware supports only 256 values with the value zero reserved for the
- * host, so we check if an assigned value belongs to a previous generation,
- * which which requires us to assign a new value. If we're the first to use a
- * VMID for the new generation, we must flush necessary caches and TLBs on all
- * CPUs.
+ * The hardware supports a limited set of values with the value zero reserved
+ * for the host, so we check if an assigned value belongs to a previous
+ * generation, which which requires us to assign a new value. If we're the
+ * first to use a VMID for the new generation, we must flush necessary caches
+ * and TLBs on all CPUs.
  */
-static bool need_new_vmid_gen(struct kvm *kvm)
+static bool need_new_vmid_gen(struct kvm_vmid *vmid)
 {
         u64 current_vmid_gen = atomic64_read(&kvm_vmid_gen);
         smp_rmb(); /* Orders read of kvm_vmid_gen and kvm->arch.vmid */
-        return unlikely(READ_ONCE(kvm->arch.vmid_gen) != current_vmid_gen);
+        return unlikely(READ_ONCE(vmid->vmid_gen) != current_vmid_gen);
 }
 
 /**
- * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
- * @kvm The guest that we are about to run
- *
- * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
- * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
- * caches and TLBs.
+ * update_vmid - Update the vmid with a valid VMID for the current generation
+ * @kvm: The guest that struct vmid belongs to
+ * @vmid: The stage-2 VMID information struct
  */
-static void update_vttbr(struct kvm *kvm)
+static void update_vmid(struct kvm_vmid *vmid)
 {
-        phys_addr_t pgd_phys;
-        u64 vmid, cnp = kvm_cpu_has_cnp() ? VTTBR_CNP_BIT : 0;
-
-        if (!need_new_vmid_gen(kvm))
+        if (!need_new_vmid_gen(vmid))
                 return;
 
         spin_lock(&kvm_vmid_lock);
@@ -512,7 +505,7 @@ static void update_vttbr(struct kvm *kvm)
          * already allocated a valid vmid for this vm, then this vcpu should
          * use the same vmid.
          */
-        if (!need_new_vmid_gen(kvm)) {
+        if (!need_new_vmid_gen(vmid)) {
                 spin_unlock(&kvm_vmid_lock);
                 return;
         }
@@ -536,18 +529,12 @@ static void update_vttbr(struct kvm *kvm)
                 kvm_call_hyp(__kvm_flush_vm_context);
         }
 
-        kvm->arch.vmid = kvm_next_vmid;
+        vmid->vmid = kvm_next_vmid;
         kvm_next_vmid++;
-        kvm_next_vmid &= (1 << kvm_vmid_bits) - 1;
-
-        /* update vttbr to be used with the new vmid */
-        pgd_phys = virt_to_phys(kvm->arch.pgd);
-        BUG_ON(pgd_phys & ~kvm_vttbr_baddr_mask(kvm));
-        vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK(kvm_vmid_bits);
-        kvm->arch.vttbr = kvm_phys_to_vttbr(pgd_phys) | vmid | cnp;
+        kvm_next_vmid &= (1 << kvm_get_vmid_bits()) - 1;
 
         smp_wmb();
-        WRITE_ONCE(kvm->arch.vmid_gen, atomic64_read(&kvm_vmid_gen));
+        WRITE_ONCE(vmid->vmid_gen, atomic64_read(&kvm_vmid_gen));
 
         spin_unlock(&kvm_vmid_lock);
 }
@@ -690,7 +677,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
                  */
                 cond_resched();
 
-                update_vttbr(vcpu->kvm);
+                update_vmid(&vcpu->kvm->arch.vmid);
 
                 check_vcpu_requests(vcpu);
 
@@ -739,7 +726,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
                  */
                 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
 
-                if (ret <= 0 || need_new_vmid_gen(vcpu->kvm) ||
+                if (ret <= 0 || need_new_vmid_gen(&vcpu->kvm->arch.vmid) ||
                     kvm_request_pending(vcpu)) {
                         vcpu->mode = OUTSIDE_GUEST_MODE;
                         isb(); /* Ensure work in x_flush_hwstate is committed */
@@ -1417,10 +1404,6 @@ static inline void hyp_cpu_pm_exit(void)
 
 static int init_common_resources(void)
 {
-        /* set size of VMID supported by CPU */
-        kvm_vmid_bits = kvm_get_vmid_bits();
-        kvm_info("%d-bit VMID\n", kvm_vmid_bits);
-
         kvm_set_ipa_limit();
 
         return 0;
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index fbdf3ac2f001..f8dda452ea24 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -908,6 +908,7 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
  */
 int kvm_alloc_stage2_pgd(struct kvm *kvm)
 {
+        phys_addr_t pgd_phys;
         pgd_t *pgd;
 
         if (kvm->arch.pgd != NULL) {
@@ -920,7 +921,12 @@ int kvm_alloc_stage2_pgd(struct kvm *kvm)
         if (!pgd)
                 return -ENOMEM;
 
+        pgd_phys = virt_to_phys(pgd);
+        if (WARN_ON(pgd_phys & ~kvm_vttbr_baddr_mask(kvm)))
+                return -EINVAL;
+
         kvm->arch.pgd = pgd;
+        kvm->arch.pgd_phys = pgd_phys;
         return 0;
 }
 
@@ -1008,6 +1014,7 @@ void kvm_free_stage2_pgd(struct kvm *kvm)
                 unmap_stage2_range(kvm, 0, kvm_phys_size(kvm));
                 pgd = READ_ONCE(kvm->arch.pgd);
                 kvm->arch.pgd = NULL;
+                kvm->arch.pgd_phys = 0;
         }
         spin_unlock(&kvm->mmu_lock);