drm/amdkfd: Add DMABuf import functionality

This is used for interoperability between ROCm compute and graphics
APIs. It allows importing graphics driver BOs into the ROCm SVM
address space for zero-copy GPU access.

The API is split into two steps (query and import) to allow user mode
to manage the virtual address space allocation for the imported buffer.

Acked-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

5 files changed, 126 insertions, 3 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 68b29a210eaa..68e4cf1b655c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -26,6 +26,7 @@
 #include "amdgpu.h"
 #include "amdgpu_gfx.h"
 #include <linux/module.h>
+#include <linux/dma-buf.h>
 
 const struct kgd2kfd_calls *kgd2kfd;
 
@@ -433,6 +434,62 @@ void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info)
         cu_info->lds_size = acu_info.lds_size;
 }
 
+int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
+                                  struct kgd_dev **dma_buf_kgd,
+                                  uint64_t *bo_size, void *metadata_buffer,
+                                  size_t buffer_size, uint32_t *metadata_size,
+                                  uint32_t *flags)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+        struct dma_buf *dma_buf;
+        struct drm_gem_object *obj;
+        struct amdgpu_bo *bo;
+        uint64_t metadata_flags;
+        int r = -EINVAL;
+
+        dma_buf = dma_buf_get(dma_buf_fd);
+        if (IS_ERR(dma_buf))
+                return PTR_ERR(dma_buf);
+
+        if (dma_buf->ops != &amdgpu_dmabuf_ops)
+                /* Can't handle non-graphics buffers */
+                goto out_put;
+
+        obj = dma_buf->priv;
+        if (obj->dev->driver != adev->ddev->driver)
+                /* Can't handle buffers from different drivers */
+                goto out_put;
+
+        adev = obj->dev->dev_private;
+        bo = gem_to_amdgpu_bo(obj);
+        if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
+                                    AMDGPU_GEM_DOMAIN_GTT)))
+                /* Only VRAM and GTT BOs are supported */
+                goto out_put;
+
+        r = 0;
+        if (dma_buf_kgd)
+                *dma_buf_kgd = (struct kgd_dev *)adev;
+        if (bo_size)
+                *bo_size = amdgpu_bo_size(bo);
+        if (metadata_size)
+                *metadata_size = bo->metadata_size;
+        if (metadata_buffer)
+                r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size,
+                                           metadata_size, &metadata_flags);
+        if (flags) {
+                *flags = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
+                        ALLOC_MEM_FLAGS_VRAM : ALLOC_MEM_FLAGS_GTT;
+
+                if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
+                        *flags |= ALLOC_MEM_FLAGS_PUBLIC;
+        }
+
+out_put:
+        dma_buf_put(dma_buf);
+        return r;
+}
+
 uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 131c6e5e6f10..70429f7aa9a8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -149,6 +149,11 @@ uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
 
 uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
 void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
+int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
+                                  struct kgd_dev **dmabuf_kgd,
+                                  uint64_t *bo_size, void *metadata_buffer,
+                                  size_t buffer_size, uint32_t *metadata_size,
+                                  uint32_t *flags);
 uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
 uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
 
@@ -200,6 +205,12 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
                                               struct kfd_vm_fault_info *info);
 
+int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
+                                      struct dma_buf *dmabuf,
+                                      uint64_t va, void *vm,
+                                      struct kgd_mem **mem, uint64_t *size,
+                                      uint64_t *mmap_offset);
+
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
 void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 5fb60e1d713a..a0a500d45886 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -25,6 +25,7 @@
 #include <linux/list.h>
 #include <linux/pagemap.h>
 #include <linux/sched/mm.h>
+#include <linux/dma-buf.h>
 #include <drm/drmP.h>
 #include "amdgpu_object.h"
 #include "amdgpu_vm.h"
@@ -1664,6 +1665,60 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
         return 0;
 }
 
+int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
+                                      struct dma_buf *dma_buf,
+                                      uint64_t va, void *vm,
+                                      struct kgd_mem **mem, uint64_t *size,
+                                      uint64_t *mmap_offset)
+{
+        struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+        struct drm_gem_object *obj;
+        struct amdgpu_bo *bo;
+        struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+
+        if (dma_buf->ops != &amdgpu_dmabuf_ops)
+                /* Can't handle non-graphics buffers */
+                return -EINVAL;
+
+        obj = dma_buf->priv;
+        if (obj->dev->dev_private != adev)
+                /* Can't handle buffers from other devices */
+                return -EINVAL;
+
+        bo = gem_to_amdgpu_bo(obj);
+        if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
+                                    AMDGPU_GEM_DOMAIN_GTT)))
+                /* Only VRAM and GTT BOs are supported */
+                return -EINVAL;
+
+        *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
+        if (!*mem)
+                return -ENOMEM;
+
+        if (size)
+                *size = amdgpu_bo_size(bo);
+
+        if (mmap_offset)
+                *mmap_offset = amdgpu_bo_mmap_offset(bo);
+
+        INIT_LIST_HEAD(&(*mem)->bo_va_list);
+        mutex_init(&(*mem)->lock);
+        (*mem)->mapping_flags =
+                AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
+                AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_NC;
+
+        (*mem)->bo = amdgpu_bo_ref(bo);
+        (*mem)->va = va;
+        (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
+                AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
+        (*mem)->mapped_to_gpu_memory = 0;
+        (*mem)->process_info = avm->process_info;
+        add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
+        amdgpu_sync_create(&(*mem)->sync);
+
+        return 0;
+}
+
 /* Evict a userptr BO by stopping the queues if necessary
  *
  * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
index d63daba9b17c..f1ddfc50bcc7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.h
@@ -54,6 +54,8 @@ void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 
+extern const struct dma_buf_ops amdgpu_dmabuf_ops;
+
 /*
  * GEM objects.
  */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
index 3e44d889f7af..71913a18d142 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_prime.c
@@ -39,8 +39,6 @@
 #include <drm/amdgpu_drm.h>
 #include <linux/dma-buf.h>
 
-static const struct dma_buf_ops amdgpu_dmabuf_ops;
-
 /**
  * amdgpu_gem_prime_get_sg_table - &drm_driver.gem_prime_get_sg_table
  * implementation
@@ -332,7 +330,7 @@ static int amdgpu_gem_begin_cpu_access(struct dma_buf *dma_buf,
         return ret;
 }
 
-static const struct dma_buf_ops amdgpu_dmabuf_ops = {
+const struct dma_buf_ops amdgpu_dmabuf_ops = {
         .attach = amdgpu_gem_map_attach,
         .detach = amdgpu_gem_map_detach,
         .map_dma_buf = drm_gem_map_dma_buf,