drm/amdkfd: Add dGPU support to the MQD manager

On dGPUs don't set ATC addressing bits and use MTYPE_UC for coherent
memory.

Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Acked-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>

4 files changed, 64 insertions, 3 deletions

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
index dfd260ef81ff..ee7061e1c466 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c
@@ -29,8 +29,15 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
         switch (dev->device_info->asic_family) {
         case CHIP_KAVERI:
                 return mqd_manager_init_cik(type, dev);
+        case CHIP_HAWAII:
+                return mqd_manager_init_cik_hawaii(type, dev);
         case CHIP_CARRIZO:
                 return mqd_manager_init_vi(type, dev);
+        case CHIP_TONGA:
+        case CHIP_FIJI:
+        case CHIP_POLARIS10:
+        case CHIP_POLARIS11:
+                return mqd_manager_init_vi_tonga(type, dev);
         default:
                 WARN(1, "Unexpected ASIC family %u",
                      dev->device_info->asic_family);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index f8ef4a051e08..fbe3f83ba685 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -170,14 +170,19 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
                                                mms);
 }
 
-static int update_mqd(struct mqd_manager *mm, void *mqd,
-                        struct queue_properties *q)
+static int __update_mqd(struct mqd_manager *mm, void *mqd,
+                        struct queue_properties *q, unsigned int atc_bit)
 {
         struct cik_mqd *m;
 
         m = get_mqd(mqd);
         m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE |
-                                DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN;
+                                DEFAULT_MIN_AVAIL_SIZE;
+        m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE;
+        if (atc_bit) {
+                m->cp_hqd_pq_control |= PQ_ATC_EN;
+                m->cp_hqd_ib_control |= IB_ATC_EN;
+        }
 
         /*
          * Calculating queue size which is log base 2 of actual queue size -1
@@ -202,6 +207,18 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
         return 0;
 }
 
+static int update_mqd(struct mqd_manager *mm, void *mqd,
+                        struct queue_properties *q)
+{
+        return __update_mqd(mm, mqd, q, 1);
+}
+
+static int update_mqd_hawaii(struct mqd_manager *mm, void *mqd,
+                        struct queue_properties *q)
+{
+        return __update_mqd(mm, mqd, q, 0);
+}
+
 static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
                                 struct queue_properties *q)
 {
@@ -441,3 +458,15 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
         return mqd;
 }
 
+struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
+                        struct kfd_dev *dev)
+{
+        struct mqd_manager *mqd;
+
+        mqd = mqd_manager_init_cik(type, dev);
+        if (!mqd)
+                return NULL;
+        if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
+                mqd->update_mqd = update_mqd_hawaii;
+        return mqd;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 971aec0637dc..58221c1fc917 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -151,6 +151,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 
         m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
         m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+        m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr);
+        m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr);
 
         m->cp_hqd_pq_doorbell_control =
                 q->doorbell_off <<
@@ -208,6 +210,12 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
         return __update_mqd(mm, mqd, q, MTYPE_CC, 1);
 }
 
+static int update_mqd_tonga(struct mqd_manager *mm, void *mqd,
+                        struct queue_properties *q)
+{
+        return __update_mqd(mm, mqd, q, MTYPE_UC, 0);
+}
+
 static int destroy_mqd(struct mqd_manager *mm, void *mqd,
                         enum kfd_preempt_type type,
                         unsigned int timeout, uint32_t pipe_id,
@@ -432,3 +440,16 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 
         return mqd;
 }
+
+struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
+                        struct kfd_dev *dev)
+{
+        struct mqd_manager *mqd;
+
+        mqd = mqd_manager_init_vi(type, dev);
+        if (!mqd)
+                return NULL;
+        if ((type == KFD_MQD_TYPE_CP) || (type == KFD_MQD_TYPE_COMPUTE))
+                mqd->update_mqd = update_mqd_tonga;
+        return mqd;
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index e5b16209f069..594f85355397 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -706,8 +706,12 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
                                         struct kfd_dev *dev);
 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
                 struct kfd_dev *dev);
+struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
+                struct kfd_dev *dev);
 struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
                 struct kfd_dev *dev);
+struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
+                struct kfd_dev *dev);
 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
 void device_queue_manager_uninit(struct device_queue_manager *dqm);
 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,