drm/amdgpu: move IV prescreening into the GMC code

The GMC/VM subsystem is causing the faults, so move the handling here as
well.

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Acked-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

9 files changed, 59 insertions, 154 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index 9ce8c93ec19b..f877bb78d10a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -51,14 +51,12 @@ struct amdgpu_ih_ring {
 struct amdgpu_ih_funcs {
         /* ring read/write ptr handling, called from interrupt context */
         u32 (*get_wptr)(struct amdgpu_device *adev);
-        bool (*prescreen_iv)(struct amdgpu_device *adev);
         void (*decode_iv)(struct amdgpu_device *adev,
                           struct amdgpu_iv_entry *entry);
         void (*set_rptr)(struct amdgpu_device *adev);
 };
 
 #define amdgpu_ih_get_wptr(adev) (adev)->irq.ih_funcs->get_wptr((adev))
-#define amdgpu_ih_prescreen_iv(adev) (adev)->irq.ih_funcs->prescreen_iv((adev))
 #define amdgpu_ih_decode_iv(adev, iv) (adev)->irq.ih_funcs->decode_iv((adev), (iv))
 #define amdgpu_ih_set_rptr(adev) (adev)->irq.ih_funcs->set_rptr((adev))
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 79b6f456f2c5..b7968f426862 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -145,10 +145,6 @@ static void amdgpu_irq_callback(struct amdgpu_device *adev,
         u32 ring_index = ih->rptr >> 2;
         struct amdgpu_iv_entry entry;
 
-        /* Prescreening of high-frequency interrupts */
-        if (!amdgpu_ih_prescreen_iv(adev))
-                return;
-
         entry.iv_entry = (const uint32_t *)&ih->ring[ring_index];
         amdgpu_ih_decode_iv(adev, &entry);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index 3e6c8c4067cb..8a8b4967a101 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -228,18 +228,6 @@ static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
  * [127:96] - reserved
  */
 
-/**
- * cik_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool cik_ih_prescreen_iv(struct amdgpu_device *adev)
-{
-        return true;
-}
-
  /**
  * cik_ih_decode_iv - decode an interrupt vector
  *
@@ -445,7 +433,6 @@ static const struct amd_ip_funcs cik_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs cik_ih_funcs = {
         .get_wptr = cik_ih_get_wptr,
-        .prescreen_iv = cik_ih_prescreen_iv,
         .decode_iv = cik_ih_decode_iv,
         .set_rptr = cik_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index 447b3cbc47e5..9d3ea298e116 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -208,18 +208,6 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
- * cz_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool cz_ih_prescreen_iv(struct amdgpu_device *adev)
-{
-        return true;
-}
-
-/**
  * cz_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -426,7 +414,6 @@ static const struct amd_ip_funcs cz_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs cz_ih_funcs = {
         .get_wptr = cz_ih_get_wptr,
-        .prescreen_iv = cz_ih_prescreen_iv,
         .decode_iv = cz_ih_decode_iv,
         .set_rptr = cz_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 016c7aab4a29..ce150de723c9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -244,6 +244,62 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
         return 0;
 }
 
+/**
+ * vega10_ih_prescreen_iv - prescreen an interrupt vector
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns true if the interrupt vector should be further processed.
+ */
+static bool gmc_v9_0_prescreen_iv(struct amdgpu_device *adev,
+                                  struct amdgpu_iv_entry *entry,
+                                  uint64_t addr)
+{
+        struct amdgpu_vm *vm;
+        u64 key;
+        int r;
+
+        /* No PASID, can't identify faulting process */
+        if (!entry->pasid)
+                return true;
+
+        /* Not a retry fault */
+        if (!(entry->src_data[1] & 0x80))
+                return true;
+
+        /* Track retry faults in per-VM fault FIFO. */
+        spin_lock(&adev->vm_manager.pasid_lock);
+        vm = idr_find(&adev->vm_manager.pasid_idr, entry->pasid);
+        if (!vm) {
+                /* VM not found, process it normally */
+                spin_unlock(&adev->vm_manager.pasid_lock);
+                return true;
+        }
+
+        key = AMDGPU_VM_FAULT(entry->pasid, addr);
+        r = amdgpu_vm_add_fault(vm->fault_hash, key);
+
+        /* Hash table is full or the fault is already being processed,
+         * ignore further page faults
+         */
+        if (r != 0) {
+                spin_unlock(&adev->vm_manager.pasid_lock);
+                return false;
+        }
+        /* No locking required with single writer and single reader */
+        r = kfifo_put(&vm->faults, key);
+        if (!r) {
+                /* FIFO is full. Ignore it until there is space */
+                amdgpu_vm_clear_fault(vm->fault_hash, key);
+                spin_unlock(&adev->vm_manager.pasid_lock);
+                return false;
+        }
+
+        spin_unlock(&adev->vm_manager.pasid_lock);
+        /* It's the first fault for this address, process it normally */
+        return true;
+}
+
 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
                                 struct amdgpu_irq_src *source,
                                 struct amdgpu_iv_entry *entry)
@@ -255,6 +311,9 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
         addr = (u64)entry->src_data[0] << 12;
         addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 
+        if (!gmc_v9_0_prescreen_iv(adev, entry, addr))
+                return 1; /* This also prevents sending it to KFD */
+
         if (!amdgpu_sriov_vf(adev)) {
                 status = RREG32(hub->vm_l2_pro_fault_status);
                 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index 2b94a6d1550e..a3984d10b604 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -208,18 +208,6 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
- * iceland_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool iceland_ih_prescreen_iv(struct amdgpu_device *adev)
-{
-        return true;
-}
-
-/**
  * iceland_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -424,7 +412,6 @@ static const struct amd_ip_funcs iceland_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs iceland_ih_funcs = {
         .get_wptr = iceland_ih_get_wptr,
-        .prescreen_iv = iceland_ih_prescreen_iv,
         .decode_iv = iceland_ih_decode_iv,
         .set_rptr = iceland_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index b3d7d9f83202..2938fb9f17cc 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -118,19 +118,6 @@ static u32 si_ih_get_wptr(struct amdgpu_device *adev)
         return (wptr & adev->irq.ih.ptr_mask);
 }
 
-/**
- * si_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool si_ih_prescreen_iv(struct amdgpu_device *adev)
-{
-        /* Process all interrupts */
-        return true;
-}
-
 static void si_ih_decode_iv(struct amdgpu_device *adev,
                              struct amdgpu_iv_entry *entry)
 {
@@ -301,7 +288,6 @@ static const struct amd_ip_funcs si_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs si_ih_funcs = {
         .get_wptr = si_ih_get_wptr,
-        .prescreen_iv = si_ih_prescreen_iv,
         .decode_iv = si_ih_decode_iv,
         .set_rptr = si_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index 9d7b43da6acc..15da06ddeb75 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -219,18 +219,6 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
- * tonga_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev)
-{
-        return true;
-}
-
-/**
  * tonga_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -490,7 +478,6 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs tonga_ih_funcs = {
         .get_wptr = tonga_ih_get_wptr,
-        .prescreen_iv = tonga_ih_prescreen_iv,
         .decode_iv = tonga_ih_decode_iv,
         .set_rptr = tonga_ih_set_rptr
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index b49290bcf109..2c250b01a903 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -220,87 +220,6 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
 }
 
 /**
- * vega10_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
-{
-        u32 ring_index = adev->irq.ih.rptr >> 2;
-        u32 dw0, dw3, dw4, dw5;
-        u16 pasid;
-        u64 addr, key;
-        struct amdgpu_vm *vm;
-        int r;
-
-        dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
-        dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
-        dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
-        dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
-
-        /* Filter retry page faults, let only the first one pass. If
-         * there are too many outstanding faults, ignore them until
-         * some faults get cleared.
-         */
-        switch (dw0 & 0xff) {
-        case SOC15_IH_CLIENTID_VMC:
-        case SOC15_IH_CLIENTID_UTCL2:
-                break;
-        default:
-                /* Not a VM fault */
-                return true;
-        }
-
-        pasid = dw3 & 0xffff;
-        /* No PASID, can't identify faulting process */
-        if (!pasid)
-                return true;
-
-        /* Not a retry fault */
-        if (!(dw5 & 0x80))
-                return true;
-
-        /* Track retry faults in per-VM fault FIFO. */
-        spin_lock(&adev->vm_manager.pasid_lock);
-        vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
-        addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
-        key = AMDGPU_VM_FAULT(pasid, addr);
-        if (!vm) {
-                /* VM not found, process it normally */
-                spin_unlock(&adev->vm_manager.pasid_lock);
-                return true;
-        } else {
-                r = amdgpu_vm_add_fault(vm->fault_hash, key);
-
-                /* Hash table is full or the fault is already being processed,
-                 * ignore further page faults
-                 */
-                if (r != 0) {
-                        spin_unlock(&adev->vm_manager.pasid_lock);
-                        goto ignore_iv;
-                }
-        }
-        /* No locking required with single writer and single reader */
-        r = kfifo_put(&vm->faults, key);
-        if (!r) {
-                /* FIFO is full. Ignore it until there is space */
-                amdgpu_vm_clear_fault(vm->fault_hash, key);
-                spin_unlock(&adev->vm_manager.pasid_lock);
-                goto ignore_iv;
-        }
-
-        spin_unlock(&adev->vm_manager.pasid_lock);
-        /* It's the first fault for this address, process it normally */
-        return true;
-
-ignore_iv:
-        adev->irq.ih.rptr += 32;
-        return false;
-}
-
-/**
  * vega10_ih_decode_iv - decode an interrupt vector
  *
  * @adev: amdgpu_device pointer
@@ -484,7 +403,6 @@ const struct amd_ip_funcs vega10_ih_ip_funcs = {
 
 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
         .get_wptr = vega10_ih_get_wptr,
-        .prescreen_iv = vega10_ih_prescreen_iv,
         .decode_iv = vega10_ih_decode_iv,
         .set_rptr = vega10_ih_set_rptr
 };