12 files changed, 158 insertions, 143 deletions

diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 9e350fd34504..9c486fad3f9f 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -626,6 +626,13 @@ static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
                 /* Awaken to handle a signal, request we sleep again later. */
                 kvm_make_request(KVM_REQ_SLEEP, vcpu);
         }
+
+        /*
+         * Make sure we will observe a potential reset request if we've
+         * observed a change to the power state. Pairs with the smp_wmb() in
+         * kvm_psci_vcpu_on().
+         */
+        smp_rmb();
 }
 
 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
@@ -639,6 +646,9 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu)
                 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
                         vcpu_req_sleep(vcpu);
 
+                if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
+                        kvm_reset_vcpu(vcpu);
+
                 /*
                  * Clear IRQ_PENDING requests that were made to guarantee
                  * that a VCPU sees new virtual interrupts.
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index fbdf3ac2f001..30251e288629 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -1695,11 +1695,14 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 
         vma_pagesize = vma_kernel_pagesize(vma);
         /*
-         * PUD level may not exist for a VM but PMD is guaranteed to
-         * exist.
+         * The stage2 has a minimum of 2 level table (For arm64 see
+         * kvm_arm_setup_stage2()). Hence, we are guaranteed that we can
+         * use PMD_SIZE huge mappings (even when the PMD is folded into PGD).
+         * As for PUD huge maps, we must make sure that we have at least
+         * 3 levels, i.e, PMD is not folded.
          */
         if ((vma_pagesize == PMD_SIZE ||
-             (vma_pagesize == PUD_SIZE && kvm_stage2_has_pud(kvm))) &&
+             (vma_pagesize == PUD_SIZE && kvm_stage2_has_pmd(kvm))) &&
             !force_pte) {
                 gfn = (fault_ipa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT;
         }
diff --git a/virt/kvm/arm/psci.c b/virt/kvm/arm/psci.c
index 9b73d3ad918a..34d08ee63747 100644
--- a/virt/kvm/arm/psci.c
+++ b/virt/kvm/arm/psci.c
@@ -104,12 +104,10 @@ static void kvm_psci_vcpu_off(struct kvm_vcpu *vcpu)
 
 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
 {
+        struct vcpu_reset_state *reset_state;
         struct kvm *kvm = source_vcpu->kvm;
         struct kvm_vcpu *vcpu = NULL;
-        struct swait_queue_head *wq;
         unsigned long cpu_id;
-        unsigned long context_id;
-        phys_addr_t target_pc;
 
         cpu_id = smccc_get_arg1(source_vcpu) & MPIDR_HWID_BITMASK;
         if (vcpu_mode_is_32bit(source_vcpu))
@@ -130,32 +128,30 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
                         return PSCI_RET_INVALID_PARAMS;
         }
 
-        target_pc = smccc_get_arg2(source_vcpu);
-        context_id = smccc_get_arg3(source_vcpu);
+        reset_state = &vcpu->arch.reset_state;
 
-        kvm_reset_vcpu(vcpu);
-
-        /* Gracefully handle Thumb2 entry point */
-        if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) {
-                target_pc &= ~((phys_addr_t) 1);
-                vcpu_set_thumb(vcpu);
-        }
+        reset_state->pc = smccc_get_arg2(source_vcpu);
 
         /* Propagate caller endianness */
-        if (kvm_vcpu_is_be(source_vcpu))
-                kvm_vcpu_set_be(vcpu);
+        reset_state->be = kvm_vcpu_is_be(source_vcpu);
 
-        *vcpu_pc(vcpu) = target_pc;
         /*
          * NOTE: We always update r0 (or x0) because for PSCI v0.1
          * the general puspose registers are undefined upon CPU_ON.
          */
-        smccc_set_retval(vcpu, context_id, 0, 0, 0);
-        vcpu->arch.power_off = false;
-        smp_mb();               /* Make sure the above is visible */
+        reset_state->r0 = smccc_get_arg3(source_vcpu);
+
+        WRITE_ONCE(reset_state->reset, true);
+        kvm_make_request(KVM_REQ_VCPU_RESET, vcpu);
 
-        wq = kvm_arch_vcpu_wq(vcpu);
-        swake_up_one(wq);
+        /*
+         * Make sure the reset request is observed if the change to
+         * power_state is observed.
+         */
+        smp_wmb();
+
+        vcpu->arch.power_off = false;
+        kvm_vcpu_wake_up(vcpu);
 
         return PSCI_RET_SUCCESS;
 }
diff --git a/virt/kvm/arm/vgic/vgic-debug.c b/virt/kvm/arm/vgic/vgic-debug.c
index 07aa900bac56..1f62f2b8065d 100644
--- a/virt/kvm/arm/vgic/vgic-debug.c
+++ b/virt/kvm/arm/vgic/vgic-debug.c
@@ -251,9 +251,9 @@ static int vgic_debug_show(struct seq_file *s, void *v)
                 return 0;
         }
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         print_irq_state(s, irq, vcpu);
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
         vgic_put_irq(kvm, irq);
         return 0;
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c
index c0c0b88af1d5..3bdb31eaed64 100644
--- a/virt/kvm/arm/vgic/vgic-init.c
+++ b/virt/kvm/arm/vgic/vgic-init.c
@@ -64,7 +64,7 @@ void kvm_vgic_early_init(struct kvm *kvm)
         struct vgic_dist *dist = &kvm->arch.vgic;
 
         INIT_LIST_HEAD(&dist->lpi_list_head);
-        spin_lock_init(&dist->lpi_list_lock);
+        raw_spin_lock_init(&dist->lpi_list_lock);
 }
 
 /* CREATION */
@@ -171,7 +171,7 @@ static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
 
                 irq->intid = i + VGIC_NR_PRIVATE_IRQS;
                 INIT_LIST_HEAD(&irq->ap_list);
-                spin_lock_init(&irq->irq_lock);
+                raw_spin_lock_init(&irq->irq_lock);
                 irq->vcpu = NULL;
                 irq->target_vcpu = vcpu0;
                 kref_init(&irq->refcount);
@@ -206,7 +206,7 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
         vgic_cpu->sgi_iodev.base_addr = VGIC_ADDR_UNDEF;
 
         INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
-        spin_lock_init(&vgic_cpu->ap_list_lock);
+        raw_spin_lock_init(&vgic_cpu->ap_list_lock);
 
         /*
          * Enable and configure all SGIs to be edge-triggered and
@@ -216,7 +216,7 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
                 struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
 
                 INIT_LIST_HEAD(&irq->ap_list);
-                spin_lock_init(&irq->irq_lock);
+                raw_spin_lock_init(&irq->irq_lock);
                 irq->intid = i;
                 irq->vcpu = NULL;
                 irq->target_vcpu = vcpu;
@@ -231,13 +231,6 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
                         irq->config = VGIC_CONFIG_LEVEL;
                 }
 
-                /*
-                 * GICv3 can only be created via the KVM_DEVICE_CREATE API and
-                 * so we always know the emulation type at this point as it's
-                 * either explicitly configured as GICv3, or explicitly
-                 * configured as GICv2, or not configured yet which also
-                 * implies GICv2.
-                 */
                 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
                         irq->group = 1;
                 else
@@ -281,7 +274,7 @@ int vgic_init(struct kvm *kvm)
 {
         struct vgic_dist *dist = &kvm->arch.vgic;
         struct kvm_vcpu *vcpu;
-        int ret = 0, i;
+        int ret = 0, i, idx;
 
         if (vgic_initialized(kvm))
                 return 0;
@@ -298,6 +291,19 @@ int vgic_init(struct kvm *kvm)
         if (ret)
                 goto out;
 
+        /* Initialize groups on CPUs created before the VGIC type was known */
+        kvm_for_each_vcpu(idx, vcpu, kvm) {
+                struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
+
+                for (i = 0; i < VGIC_NR_PRIVATE_IRQS; i++) {
+                        struct vgic_irq *irq = &vgic_cpu->private_irqs[i];
+                        if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
+                                irq->group = 1;
+                        else
+                                irq->group = 0;
+                }
+        }
+
         if (vgic_has_its(kvm)) {
                 ret = vgic_v4_init(kvm);
                 if (ret)
diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c
index eb2a390a6c86..ab3f47745d9c 100644
--- a/virt/kvm/arm/vgic/vgic-its.c
+++ b/virt/kvm/arm/vgic/vgic-its.c
@@ -65,7 +65,7 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
 
         INIT_LIST_HEAD(&irq->lpi_list);
         INIT_LIST_HEAD(&irq->ap_list);
-        spin_lock_init(&irq->irq_lock);
+        raw_spin_lock_init(&irq->irq_lock);
 
         irq->config = VGIC_CONFIG_EDGE;
         kref_init(&irq->refcount);
@@ -73,7 +73,7 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
         irq->target_vcpu = vcpu;
         irq->group = 1;
 
-        spin_lock_irqsave(&dist->lpi_list_lock, flags);
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 
         /*
          * There could be a race with another vgic_add_lpi(), so we need to
@@ -101,7 +101,7 @@ static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
         dist->lpi_list_count++;
 
 out_unlock:
-        spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 
         /*
          * We "cache" the configuration table entries in our struct vgic_irq's.
@@ -287,7 +287,7 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
         if (ret)
                 return ret;
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
                 irq->priority = LPI_PROP_PRIORITY(prop);
@@ -299,7 +299,7 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
                 }
         }
 
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
         if (irq->hw)
                 return its_prop_update_vlpi(irq->host_irq, prop, needs_inv);
@@ -332,7 +332,7 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr)
         if (!intids)
                 return -ENOMEM;
 
-        spin_lock_irqsave(&dist->lpi_list_lock, flags);
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
                 if (i == irq_count)
                         break;
@@ -341,7 +341,7 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr)
                         continue;
                 intids[i++] = irq->intid;
         }
-        spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 
         *intid_ptr = intids;
         return i;
@@ -352,9 +352,9 @@ static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu)
         int ret = 0;
         unsigned long flags;
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         irq->target_vcpu = vcpu;
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
         if (irq->hw) {
                 struct its_vlpi_map map;
@@ -455,7 +455,7 @@ static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
                 }
 
                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 irq->pending_latch = pendmask & (1U << bit_nr);
                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
                 vgic_put_irq(vcpu->kvm, irq);
@@ -612,7 +612,7 @@ static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
                 return irq_set_irqchip_state(irq->host_irq,
                                              IRQCHIP_STATE_PENDING, true);
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         irq->pending_latch = true;
         vgic_queue_irq_unlock(kvm, irq, flags);
 
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v2.c b/virt/kvm/arm/vgic/vgic-mmio-v2.c
index 738b65d2d0e7..b535fffc7400 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v2.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v2.c
@@ -147,7 +147,7 @@ static void vgic_mmio_write_sgir(struct kvm_vcpu *source_vcpu,
 
                 irq = vgic_get_irq(source_vcpu->kvm, vcpu, intid);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 irq->pending_latch = true;
                 irq->source |= 1U << source_vcpu->vcpu_id;
 
@@ -191,13 +191,13 @@ static void vgic_mmio_write_target(struct kvm_vcpu *vcpu,
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, NULL, intid + i);
                 int target;
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 irq->targets = (val >> (i * 8)) & cpu_mask;
                 target = irq->targets ? __ffs(irq->targets) : 0;
                 irq->target_vcpu = kvm_get_vcpu(vcpu->kvm, target);
 
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 }
@@ -230,13 +230,13 @@ static void vgic_mmio_write_sgipendc(struct kvm_vcpu *vcpu,
         for (i = 0; i < len; i++) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 irq->source &= ~((val >> (i * 8)) & 0xff);
                 if (!irq->source)
                         irq->pending_latch = false;
 
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 }
@@ -252,7 +252,7 @@ static void vgic_mmio_write_sgipends(struct kvm_vcpu *vcpu,
         for (i = 0; i < len; i++) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 irq->source |= (val >> (i * 8)) & 0xff;
 
@@ -260,7 +260,7 @@ static void vgic_mmio_write_sgipends(struct kvm_vcpu *vcpu,
                         irq->pending_latch = true;
                         vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
                 } else {
-                        spin_unlock_irqrestore(&irq->irq_lock, flags);
+                        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 }
                 vgic_put_irq(vcpu->kvm, irq);
         }
diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c
index b3d1f0985117..4a12322bf7df 100644
--- a/virt/kvm/arm/vgic/vgic-mmio-v3.c
+++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c
@@ -169,13 +169,13 @@ static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu,
         if (!irq)
                 return;
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
         /* We only care about and preserve Aff0, Aff1 and Aff2. */
         irq->mpidr = val & GENMASK(23, 0);
         irq->target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, irq->mpidr);
 
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
         vgic_put_irq(vcpu->kvm, irq);
 }
 
@@ -281,7 +281,7 @@ static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu,
         for (i = 0; i < len * 8; i++) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 if (test_bit(i, &val)) {
                         /*
                          * pending_latch is set irrespective of irq type
@@ -292,7 +292,7 @@ static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu,
                         vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
                 } else {
                         irq->pending_latch = false;
-                        spin_unlock_irqrestore(&irq->irq_lock, flags);
+                        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 }
 
                 vgic_put_irq(vcpu->kvm, irq);
@@ -957,7 +957,7 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1)
 
                 irq = vgic_get_irq(vcpu->kvm, c_vcpu, sgi);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 /*
                  * An access targetting Group0 SGIs can only generate
@@ -968,7 +968,7 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1)
                         irq->pending_latch = true;
                         vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
                 } else {
-                        spin_unlock_irqrestore(&irq->irq_lock, flags);
+                        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 }
 
                 vgic_put_irq(vcpu->kvm, irq);
diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c
index ceeda7e04a4d..7de42fba05b5 100644
--- a/virt/kvm/arm/vgic/vgic-mmio.c
+++ b/virt/kvm/arm/vgic/vgic-mmio.c
@@ -77,7 +77,7 @@ void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr,
         for (i = 0; i < len * 8; i++) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 irq->group = !!(val & BIT(i));
                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 
@@ -120,7 +120,7 @@ void vgic_mmio_write_senable(struct kvm_vcpu *vcpu,
         for_each_set_bit(i, &val, len * 8) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 irq->enabled = true;
                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
 
@@ -139,11 +139,11 @@ void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu,
         for_each_set_bit(i, &val, len * 8) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 irq->enabled = false;
 
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 }
@@ -160,10 +160,10 @@ unsigned long vgic_mmio_read_pending(struct kvm_vcpu *vcpu,
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
                 unsigned long flags;
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 if (irq_is_pending(irq))
                         value |= (1U << i);
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
                 vgic_put_irq(vcpu->kvm, irq);
         }
@@ -215,7 +215,7 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu,
         for_each_set_bit(i, &val, len * 8) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 if (irq->hw)
                         vgic_hw_irq_spending(vcpu, irq, is_uaccess);
                 else
@@ -262,14 +262,14 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu,
         for_each_set_bit(i, &val, len * 8) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 if (irq->hw)
                         vgic_hw_irq_cpending(vcpu, irq, is_uaccess);
                 else
                         irq->pending_latch = false;
 
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 }
@@ -311,7 +311,7 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
         unsigned long flags;
         struct kvm_vcpu *requester_vcpu = vgic_get_mmio_requester_vcpu();
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
         if (irq->hw) {
                 vgic_hw_irq_change_active(vcpu, irq, active, !requester_vcpu);
@@ -342,7 +342,7 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq,
         if (irq->active)
                 vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
         else
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 }
 
 /*
@@ -485,10 +485,10 @@ void vgic_mmio_write_priority(struct kvm_vcpu *vcpu,
         for (i = 0; i < len; i++) {
                 struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 /* Narrow the priority range to what we actually support */
                 irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS);
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
                 vgic_put_irq(vcpu->kvm, irq);
         }
@@ -534,14 +534,14 @@ void vgic_mmio_write_config(struct kvm_vcpu *vcpu,
                         continue;
 
                 irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i);
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
                 if (test_bit(i * 2 + 1, &val))
                         irq->config = VGIC_CONFIG_EDGE;
                 else
                         irq->config = VGIC_CONFIG_LEVEL;
 
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 }
@@ -590,12 +590,12 @@ void vgic_write_irq_line_level_info(struct kvm_vcpu *vcpu, u32 intid,
                  * restore irq config before line level.
                  */
                 new_level = !!(val & (1U << i));
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 irq->line_level = new_level;
                 if (new_level)
                         vgic_queue_irq_unlock(vcpu->kvm, irq, flags);
                 else
-                        spin_unlock_irqrestore(&irq->irq_lock, flags);
+                        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
                 vgic_put_irq(vcpu->kvm, irq);
         }
diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c
index 69b892abd7dc..d91a8938aa7c 100644
--- a/virt/kvm/arm/vgic/vgic-v2.c
+++ b/virt/kvm/arm/vgic/vgic-v2.c
@@ -84,7 +84,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
 
                 irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
 
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&irq->irq_lock);
 
                 /* Always preserve the active bit */
                 irq->active = !!(val & GICH_LR_ACTIVE_BIT);
@@ -127,7 +127,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu)
                                 vgic_irq_set_phys_active(irq, false);
                 }
 
-                spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&irq->irq_lock);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 
diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/virt/kvm/arm/vgic/vgic-v3.c
index 9c0dd234ebe8..4ee0aeb9a905 100644
--- a/virt/kvm/arm/vgic/vgic-v3.c
+++ b/virt/kvm/arm/vgic/vgic-v3.c
@@ -76,7 +76,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
                 if (!irq)       /* An LPI could have been unmapped. */
                         continue;
 
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&irq->irq_lock);
 
                 /* Always preserve the active bit */
                 irq->active = !!(val & ICH_LR_ACTIVE_BIT);
@@ -119,7 +119,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu)
                                 vgic_irq_set_phys_active(irq, false);
                 }
 
-                spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&irq->irq_lock);
                 vgic_put_irq(vcpu->kvm, irq);
         }
 
@@ -347,9 +347,9 @@ retry:
 
         status = val & (1 << bit_nr);
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         if (irq->target_vcpu != vcpu) {
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 goto retry;
         }
         irq->pending_latch = status;
diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c
index 870b1185173b..abd9c7352677 100644
--- a/virt/kvm/arm/vgic/vgic.c
+++ b/virt/kvm/arm/vgic/vgic.c
@@ -54,11 +54,11 @@ struct vgic_global kvm_vgic_global_state __ro_after_init = {
  * When taking more than one ap_list_lock at the same time, always take the
  * lowest numbered VCPU's ap_list_lock first, so:
  *   vcpuX->vcpu_id < vcpuY->vcpu_id:
- *     spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock);
- *     spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock);
+ *     raw_spin_lock(vcpuX->arch.vgic_cpu.ap_list_lock);
+ *     raw_spin_lock(vcpuY->arch.vgic_cpu.ap_list_lock);
  *
  * Since the VGIC must support injecting virtual interrupts from ISRs, we have
- * to use the spin_lock_irqsave/spin_unlock_irqrestore versions of outer
+ * to use the raw_spin_lock_irqsave/raw_spin_unlock_irqrestore versions of outer
  * spinlocks for any lock that may be taken while injecting an interrupt.
  */
 
@@ -72,7 +72,7 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
         struct vgic_irq *irq = NULL;
         unsigned long flags;
 
-        spin_lock_irqsave(&dist->lpi_list_lock, flags);
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
 
         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
                 if (irq->intid != intid)
@@ -88,7 +88,7 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
         irq = NULL;
 
 out_unlock:
-        spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 
         return irq;
 }
@@ -138,15 +138,15 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
         if (irq->intid < VGIC_MIN_LPI)
                 return;
 
-        spin_lock_irqsave(&dist->lpi_list_lock, flags);
+        raw_spin_lock_irqsave(&dist->lpi_list_lock, flags);
         if (!kref_put(&irq->refcount, vgic_irq_release)) {
-                spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+                raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
                 return;
         };
 
         list_del(&irq->lpi_list);
         dist->lpi_list_count--;
-        spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
+        raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags);
 
         kfree(irq);
 }
@@ -244,8 +244,8 @@ static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b)
         bool penda, pendb;
         int ret;
 
-        spin_lock(&irqa->irq_lock);
-        spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
+        raw_spin_lock(&irqa->irq_lock);
+        raw_spin_lock_nested(&irqb->irq_lock, SINGLE_DEPTH_NESTING);
 
         if (irqa->active || irqb->active) {
                 ret = (int)irqb->active - (int)irqa->active;
@@ -263,8 +263,8 @@ static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b)
         /* Both pending and enabled, sort by priority */
         ret = irqa->priority - irqb->priority;
 out:
-        spin_unlock(&irqb->irq_lock);
-        spin_unlock(&irqa->irq_lock);
+        raw_spin_unlock(&irqb->irq_lock);
+        raw_spin_unlock(&irqa->irq_lock);
         return ret;
 }
 
@@ -325,7 +325,7 @@ retry:
                  * not need to be inserted into an ap_list and there is also
                  * no more work for us to do.
                  */
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
                 /*
                  * We have to kick the VCPU here, because we could be
@@ -347,12 +347,12 @@ retry:
          * We must unlock the irq lock to take the ap_list_lock where
          * we are going to insert this new pending interrupt.
          */
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
         /* someone can do stuff here, which we re-check below */
 
-        spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
-        spin_lock(&irq->irq_lock);
+        raw_spin_lock_irqsave(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
+        raw_spin_lock(&irq->irq_lock);
 
         /*
          * Did something change behind our backs?
@@ -367,10 +367,11 @@ retry:
          */
 
         if (unlikely(irq->vcpu || vcpu != vgic_target_oracle(irq))) {
-                spin_unlock(&irq->irq_lock);
-                spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
+                raw_spin_unlock(&irq->irq_lock);
+                raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock,
+                                           flags);
 
-                spin_lock_irqsave(&irq->irq_lock, flags);
+                raw_spin_lock_irqsave(&irq->irq_lock, flags);
                 goto retry;
         }
 
@@ -382,8 +383,8 @@ retry:
         list_add_tail(&irq->ap_list, &vcpu->arch.vgic_cpu.ap_list_head);
         irq->vcpu = vcpu;
 
-        spin_unlock(&irq->irq_lock);
-        spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
+        raw_spin_unlock(&irq->irq_lock);
+        raw_spin_unlock_irqrestore(&vcpu->arch.vgic_cpu.ap_list_lock, flags);
 
         kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
         kvm_vcpu_kick(vcpu);
@@ -430,11 +431,11 @@ int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid,
         if (!irq)
                 return -EINVAL;
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
 
         if (!vgic_validate_injection(irq, level, owner)) {
                 /* Nothing to see here, move along... */
-                spin_unlock_irqrestore(&irq->irq_lock, flags);
+                raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
                 vgic_put_irq(kvm, irq);
                 return 0;
         }
@@ -494,9 +495,9 @@ int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
 
         BUG_ON(!irq);
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         ret = kvm_vgic_map_irq(vcpu, irq, host_irq, get_input_level);
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
         vgic_put_irq(vcpu->kvm, irq);
 
         return ret;
@@ -519,11 +520,11 @@ void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid)
         if (!irq->hw)
                 goto out;
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         irq->active = false;
         irq->pending_latch = false;
         irq->line_level = false;
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 out:
         vgic_put_irq(vcpu->kvm, irq);
 }
@@ -539,9 +540,9 @@ int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid)
         irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
         BUG_ON(!irq);
 
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         kvm_vgic_unmap_irq(irq);
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
         vgic_put_irq(vcpu->kvm, irq);
 
         return 0;
@@ -571,12 +572,12 @@ int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner)
                 return -EINVAL;
 
         irq = vgic_get_irq(vcpu->kvm, vcpu, intid);
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         if (irq->owner && irq->owner != owner)
                 ret = -EEXIST;
         else
                 irq->owner = owner;
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
 
         return ret;
 }
@@ -597,13 +598,13 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu)
         DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
 retry:
-        spin_lock(&vgic_cpu->ap_list_lock);
+        raw_spin_lock(&vgic_cpu->ap_list_lock);
 
         list_for_each_entry_safe(irq, tmp, &vgic_cpu->ap_list_head, ap_list) {
                 struct kvm_vcpu *target_vcpu, *vcpuA, *vcpuB;
                 bool target_vcpu_needs_kick = false;
 
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&irq->irq_lock);
 
                 BUG_ON(vcpu != irq->vcpu);
 
@@ -616,7 +617,7 @@ retry:
                          */
                         list_del(&irq->ap_list);
                         irq->vcpu = NULL;
-                        spin_unlock(&irq->irq_lock);
+                        raw_spin_unlock(&irq->irq_lock);
 
                         /*
                          * This vgic_put_irq call matches the
@@ -631,14 +632,14 @@ retry:
 
                 if (target_vcpu == vcpu) {
                         /* We're on the right CPU */
-                        spin_unlock(&irq->irq_lock);
+                        raw_spin_unlock(&irq->irq_lock);
                         continue;
                 }
 
                 /* This interrupt looks like it has to be migrated. */
 
-                spin_unlock(&irq->irq_lock);
-                spin_unlock(&vgic_cpu->ap_list_lock);
+                raw_spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&vgic_cpu->ap_list_lock);
 
                 /*
                  * Ensure locking order by always locking the smallest
@@ -652,10 +653,10 @@ retry:
                         vcpuB = vcpu;
                 }
 
-                spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock);
-                spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock,
-                                 SINGLE_DEPTH_NESTING);
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&vcpuA->arch.vgic_cpu.ap_list_lock);
+                raw_spin_lock_nested(&vcpuB->arch.vgic_cpu.ap_list_lock,
+                                      SINGLE_DEPTH_NESTING);
+                raw_spin_lock(&irq->irq_lock);
 
                 /*
                  * If the affinity has been preserved, move the
@@ -675,9 +676,9 @@ retry:
                         target_vcpu_needs_kick = true;
                 }
 
-                spin_unlock(&irq->irq_lock);
-                spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
-                spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
+                raw_spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&vcpuB->arch.vgic_cpu.ap_list_lock);
+                raw_spin_unlock(&vcpuA->arch.vgic_cpu.ap_list_lock);
 
                 if (target_vcpu_needs_kick) {
                         kvm_make_request(KVM_REQ_IRQ_PENDING, target_vcpu);
@@ -687,7 +688,7 @@ retry:
                 goto retry;
         }
 
-        spin_unlock(&vgic_cpu->ap_list_lock);
+        raw_spin_unlock(&vgic_cpu->ap_list_lock);
 }
 
 static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
@@ -741,10 +742,10 @@ static int compute_ap_list_depth(struct kvm_vcpu *vcpu,
         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
                 int w;
 
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&irq->irq_lock);
                 /* GICv2 SGIs can count for more than one... */
                 w = vgic_irq_get_lr_count(irq);
-                spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&irq->irq_lock);
 
                 count += w;
                 *multi_sgi |= (w > 1);
@@ -770,7 +771,7 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
         count = 0;
 
         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&irq->irq_lock);
 
                 /*
                  * If we have multi-SGIs in the pipeline, we need to
@@ -780,7 +781,7 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
                  * the AP list has been sorted already.
                  */
                 if (multi_sgi && irq->priority > prio) {
-                        spin_unlock(&irq->irq_lock);
+                        _raw_spin_unlock(&irq->irq_lock);
                         break;
                 }
 
@@ -791,7 +792,7 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu)
                                 prio = irq->priority;
                 }
 
-                spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&irq->irq_lock);
 
                 if (count == kvm_vgic_global_state.nr_lr) {
                         if (!list_is_last(&irq->ap_list,
@@ -872,9 +873,9 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 
         DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
-        spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
+        raw_spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock);
         vgic_flush_lr_state(vcpu);
-        spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
+        raw_spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock);
 
         if (can_access_vgic_from_kernel())
                 vgic_restore_state(vcpu);
@@ -918,20 +919,20 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
 
         vgic_get_vmcr(vcpu, &vmcr);
 
-        spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
+        raw_spin_lock_irqsave(&vgic_cpu->ap_list_lock, flags);
 
         list_for_each_entry(irq, &vgic_cpu->ap_list_head, ap_list) {
-                spin_lock(&irq->irq_lock);
+                raw_spin_lock(&irq->irq_lock);
                 pending = irq_is_pending(irq) && irq->enabled &&
                           !irq->active &&
                           irq->priority < vmcr.pmr;
-                spin_unlock(&irq->irq_lock);
+                raw_spin_unlock(&irq->irq_lock);
 
                 if (pending)
                         break;
         }
 
-        spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
+        raw_spin_unlock_irqrestore(&vgic_cpu->ap_list_lock, flags);
 
         return pending;
 }
@@ -963,11 +964,10 @@ bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid)
                 return false;
 
         irq = vgic_get_irq(vcpu->kvm, vcpu, vintid);
-        spin_lock_irqsave(&irq->irq_lock, flags);
+        raw_spin_lock_irqsave(&irq->irq_lock, flags);
         map_is_active = irq->hw && irq->active;
-        spin_unlock_irqrestore(&irq->irq_lock, flags);
+        raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
         vgic_put_irq(vcpu->kvm, irq);
 
         return map_is_active;
 }
-