drm/amdgpu: Add PCIe replay count sysfs file

Add a sysfs file for reporting the number of PCIe replays (NAKs). This
returns the sum of NAKs received and NAKs generated

Signed-off-by: Kent Russell <kent.russell@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

6 files changed, 85 insertions, 0 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 30165487dabd..4b10f3c1c6c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -558,6 +558,8 @@ struct amdgpu_asic_funcs {
                                uint64_t *count1);
         /* do we need to reset the asic at init time (e.g., kexec) */
         bool (*need_reset_on_init)(struct amdgpu_device *adev);
+        /* PCIe replay counter */
+        uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev);
 };
 
 /*
@@ -1077,6 +1079,7 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
 #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev))
 #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
 #define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev))
+#define amdgpu_asic_get_pcie_replay_count(adev) ((adev)->asic_funcs->get_pcie_replay_count((adev)))
 
 /* Common functions */
 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 0237513086e8..665764ff7eb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -97,6 +97,28 @@ static const char *amdgpu_asic_name[] = {
         "LAST",
 };
 
+/**
+ * DOC: pcie_replay_count
+ *
+ * The amdgpu driver provides a sysfs API for reporting the total number
+ * of PCIe replays (NAKs)
+ * The file pcie_replay_count is used for this and returns the total
+ * number of replays as a sum of the NAKs generated and NAKs received
+ */
+
+static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
+                struct device_attribute *attr, char *buf)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
+
+        return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
+}
+
+static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
+                amdgpu_device_get_pcie_replay_count, NULL);
+
 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 
 /**
@@ -2721,6 +2743,12 @@ fence_driver_init:
         /* must succeed. */
         amdgpu_ras_post_init(adev);
 
+        r = device_create_file(adev->dev, &dev_attr_pcie_replay_count);
+        if (r) {
+                dev_err(adev->dev, "Could not create pcie_replay_count");
+                return r;
+        }
+
         return 0;
 
 failed:
@@ -2784,6 +2812,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         adev->rmmio = NULL;
         amdgpu_device_doorbell_fini(adev);
         amdgpu_debugfs_regs_cleanup(adev);
+        device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 07c1f239e9c3..3a4f20766a39 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -1804,6 +1804,18 @@ static bool cik_need_reset_on_init(struct amdgpu_device *adev)
         return false;
 }
 
+static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+        uint64_t nak_r, nak_g;
+
+        /* Get the number of NAKs received and generated */
+        nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
+        nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
+
+        /* Add the total number of NAKs, i.e the number of replays */
+        return (nak_r + nak_g);
+}
+
 static const struct amdgpu_asic_funcs cik_asic_funcs =
 {
         .read_disabled_bios = &cik_read_disabled_bios,
@@ -1821,6 +1833,7 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
         .init_doorbell_index = &legacy_doorbell_index_init,
         .get_pcie_usage = &cik_get_pcie_usage,
         .need_reset_on_init = &cik_need_reset_on_init,
+        .get_pcie_replay_count = &cik_get_pcie_replay_count,
 };
 
 static int cik_common_early_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 9d8df68893b9..4ff930a47e10 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -1375,6 +1375,18 @@ static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
 }
 
+static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+        uint64_t nak_r, nak_g;
+
+        /* Get the number of NAKs received and generated */
+        nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
+        nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
+
+        /* Add the total number of NAKs, i.e the number of replays */
+        return (nak_r + nak_g);
+}
+
 static const struct amdgpu_asic_funcs si_asic_funcs =
 {
         .read_disabled_bios = &si_read_disabled_bios,
@@ -1393,6 +1405,7 @@ static const struct amdgpu_asic_funcs si_asic_funcs =
         .need_full_reset = &si_need_full_reset,
         .get_pcie_usage = &si_get_pcie_usage,
         .need_reset_on_init = &si_need_reset_on_init,
+        .get_pcie_replay_count = &si_get_pcie_replay_count,
 };
 
 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 78bd00a0142f..78b27c03b8c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -745,6 +745,18 @@ static bool soc15_need_reset_on_init(struct amdgpu_device *adev)
         return false;
 }
 
+static uint64_t soc15_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+        uint64_t nak_r, nak_g;
+
+        /* Get the number of NAKs received and generated */
+        nak_r = RREG32_PCIE(smnPCIE_RX_NUM_NAK);
+        nak_g = RREG32_PCIE(smnPCIE_RX_NUM_NAK_GENERATED);
+
+        /* Add the total number of NAKs, i.e the number of replays */
+        return (nak_r + nak_g);
+}
+
 static const struct amdgpu_asic_funcs soc15_asic_funcs =
 {
         .read_disabled_bios = &soc15_read_disabled_bios,
@@ -762,6 +774,7 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs =
         .init_doorbell_index = &vega10_doorbell_index_init,
         .get_pcie_usage = &soc15_get_pcie_usage,
         .need_reset_on_init = &soc15_need_reset_on_init,
+        .get_pcie_replay_count = &soc15_get_pcie_replay_count,
 };
 
 static const struct amdgpu_asic_funcs vega20_asic_funcs =
@@ -781,6 +794,7 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs =
         .init_doorbell_index = &vega20_doorbell_index_init,
         .get_pcie_usage = &soc15_get_pcie_usage,
         .need_reset_on_init = &soc15_need_reset_on_init,
+        .get_pcie_replay_count = &soc15_get_pcie_replay_count,
 };
 
 static int soc15_common_early_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 5e5b42a0744a..b8adf3808de2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -987,6 +987,18 @@ static void vi_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
         *count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
 }
 
+static uint64_t vi_get_pcie_replay_count(struct amdgpu_device *adev)
+{
+        uint64_t nak_r, nak_g;
+
+        /* Get the number of NAKs received and generated */
+        nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
+        nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
+
+        /* Add the total number of NAKs, i.e the number of replays */
+        return (nak_r + nak_g);
+}
+
 static bool vi_need_reset_on_init(struct amdgpu_device *adev)
 {
         u32 clock_cntl, pc;
@@ -1021,6 +1033,7 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
         .init_doorbell_index = &legacy_doorbell_index_init,
         .get_pcie_usage = &vi_get_pcie_usage,
         .need_reset_on_init = &vi_need_reset_on_init,
+        .get_pcie_replay_count = &vi_get_pcie_replay_count,
 };
 
 #define CZ_REV_BRISTOL(rev)      \