gpu-paging: Functions to support initial working versionrtss22-aegpu-paging

- Buffer deallocation (+IOCTL)
- Buffer reallocation
- Private dmabuf user list and accessor

8 files changed, 131 insertions, 13 deletions

diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c
index 74c9e4b90..ea648b8bf 100644
--- a/drivers/video/tegra/nvmap/nvmap_alloc.c
+++ b/drivers/video/tegra/nvmap/nvmap_alloc.c
@@ -775,6 +775,7 @@ int nvmap_alloc_handle(struct nvmap_client *client,
                 nvmap_stats_read(NS_TOTAL),
                 nvmap_stats_read(NS_ALLOC));
         h->userflags = flags;
+        h->userheap_mask = heap_mask;
         nr_page = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT);
         /* Force mapping to uncached for VPR memory. */
         if (heap_mask & (NVMAP_HEAP_CARVEOUT_VPR | ~nvmap_dev->cpu_access_mask))
@@ -911,20 +912,9 @@ int nvmap_alloc_handle_from_va(struct nvmap_client *client,
         return err;
 }
 
-void _nvmap_handle_free(struct nvmap_handle *h)
+void __nvmap_handle_free(struct nvmap_handle *h)
 {
         unsigned int i, nr_page, page_index = 0;
-        struct nvmap_handle_dmabuf_priv *curr, *next;
-
-        list_for_each_entry_safe(curr, next, &h->dmabuf_priv, list) {
-                curr->priv_release(curr->priv);
-                list_del(&curr->list);
-                kzfree(curr);
-        }
-
-        if (nvmap_handle_remove(nvmap_dev, h) != 0)
-                return;
-
         if (!h->alloc)
                 goto out;
 
@@ -980,8 +970,24 @@ void _nvmap_handle_free(struct nvmap_handle *h)
         }
 
         nvmap_altfree(h->pgalloc.pages, nr_page * sizeof(struct page *));
-
 out:
+        h->alloc = false;
+}
+
+void _nvmap_handle_free(struct nvmap_handle *h)
+{
+        struct nvmap_handle_dmabuf_priv *curr, *next;
+
+        list_for_each_entry_safe(curr, next, &h->dmabuf_priv, list) {
+                curr->priv_release(curr->priv);
+                list_del(&curr->list);
+                kzfree(curr);
+        }
+
+        if (nvmap_handle_remove(nvmap_dev, h) != 0)
+                return;
+        __nvmap_handle_free(h);
+        // Next two calls were originally in the `out` clause
         NVMAP_TAG_TRACE(trace_nvmap_destroy_handle,
                 NULL, get_current()->pid, 0, NVMAP_TP_ARGS_H(h));
         kfree(h);
@@ -1041,3 +1047,57 @@ void nvmap_free_handle_fd(struct nvmap_client *client,
                 nvmap_handle_put(handle);
         }
 }
+
+// Frees backing pages. Handle remains valid.
+void nvmap_dealloc_fd(struct nvmap_client *client, int fd)
+{
+        struct nvmap_handle *handle = nvmap_handle_get_from_fd(fd);
+        if (handle)
+                __nvmap_handle_free(handle);
+}
+
+// For deallocations from elsewhere in the kernel.
+// (Userspace should use NVMAP_IOC_DEALLOC ioctl.)
+int nvmap_dealloc_dmabuf(struct dma_buf *dmabuf) {
+        struct nvmap_handle_info *info;
+
+        if (!dmabuf_is_nvmap(dmabuf))
+                return -EINVAL;
+        info = dmabuf->priv;
+        // Fail if mapped into kernelspace
+        // (Unclear if this is necessary, but no handles have been observed with this
+        // set, and so testing has not been possible.)
+        if (atomic_read(&info->handle->kmap_count))
+                goto out_busy;
+        // Buffer cannot currently be mapped into userspace (via mmap)
+        if (atomic_read(&info->handle->umap_count))
+                goto out_busy;
+        // Other devices should not still have this pinned into DMA memory
+        if (atomic_read(&info->handle->pin))
+                goto out_busy;
+        __nvmap_handle_free(info->handle);
+        return 0;
+out_busy:
+        printk(KERN_WARNING "nvmap: Unmap requested on FD that is currently in-use! (%d h->pin, %d h->kmap_count, %d h->umap_count)\n",
+               atomic_read(&info->handle->pin),
+               atomic_read(&info->handle->kmap_count),
+               atomic_read(&info->handle->umap_count));
+        return -EBUSY;
+}
+EXPORT_SYMBOL(nvmap_dealloc_dmabuf);
+
+// For reallocations from elsewhere in the kernel.
+// (Userspace should use NVMAP_IOC_ALLOC ioctl.)
+int nvmap_realloc_dmabuf(struct dma_buf *dmabuf) {
+        struct nvmap_handle_info *info;
+        struct nvmap_handle *h;
+
+        if (!dmabuf_is_nvmap(dmabuf))
+                return -EINVAL;
+        info = dmabuf->priv;
+        h = info->handle;
+        return nvmap_alloc_handle(h->owner, h, h->userheap_mask,
+                                  h->align, 0, h->userflags,
+                                  NVMAP_IVM_INVALID_PEER);
+}
+EXPORT_SYMBOL(nvmap_realloc_dmabuf);
diff --git a/drivers/video/tegra/nvmap/nvmap_dev.c b/drivers/video/tegra/nvmap/nvmap_dev.c
index eb5791464..a11e3edfb 100644
--- a/drivers/video/tegra/nvmap/nvmap_dev.c
+++ b/drivers/video/tegra/nvmap/nvmap_dev.c
@@ -360,6 +360,10 @@ static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
                 err = nvmap_ioctl_alloc(filp, uarg);
                 break;
 
+        case NVMAP_IOC_DEALLOC:
+                err = nvmap_ioctl_dealloc(filp, arg);
+                break;
+
         case NVMAP_IOC_ALLOC_IVM:
                 err = nvmap_ioctl_alloc_ivm(filp, uarg);
                 break;
diff --git a/drivers/video/tegra/nvmap/nvmap_dmabuf.c b/drivers/video/tegra/nvmap/nvmap_dmabuf.c
index 850e07e06..33dcf5fe5 100644
--- a/drivers/video/tegra/nvmap/nvmap_dmabuf.c
+++ b/drivers/video/tegra/nvmap/nvmap_dmabuf.c
@@ -659,6 +659,7 @@ struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client,
         info->handle = handle;
         INIT_LIST_HEAD(&info->maps);
         mutex_init(&info->maps_lock);
+        INIT_LIST_HEAD(&info->user_priv_list);
 
         dmabuf = __dma_buf_export(info, handle->size, ro_buf);
         if (IS_ERR(dmabuf)) {
@@ -791,3 +792,13 @@ error:
         dma_buf_put(dmabuf);
         return ret;
 }
+
+struct list_head* nvmap_get_priv_list(struct dma_buf *dmabuf) {
+        struct nvmap_handle_info *info;
+
+        if (!dmabuf_is_nvmap(dmabuf))
+                return ERR_PTR(-EINVAL);
+        info = dmabuf->priv;
+        return &info->user_priv_list;
+}
+EXPORT_SYMBOL(nvmap_get_priv_list);
diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c
index 7fd13e625..93a418eb3 100644
--- a/drivers/video/tegra/nvmap/nvmap_ioctl.c
+++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c
@@ -460,6 +460,19 @@ int nvmap_ioctl_free(struct file *filp, unsigned long arg)
         return sys_close(arg);
 }
 
+// Returns memory freed by deallocation
+int nvmap_ioctl_dealloc(struct file *filp, unsigned long arg)
+{
+        struct nvmap_client *client = filp->private_data;
+        u64 before = nvmap_stats_read(NS_TOTAL);
+
+        if (!arg)
+                return 0;
+
+        nvmap_dealloc_fd(client, arg);
+        return before - nvmap_stats_read(NS_TOTAL);
+}
+
 static ssize_t rw_handle(struct nvmap_client *client, struct nvmap_handle *h,
                          int is_read, unsigned long h_offs,
                          unsigned long sys_addr, unsigned long h_stride,
diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.h b/drivers/video/tegra/nvmap/nvmap_ioctl.h
index 6c64d2abe..f1b7b9e82 100644
--- a/drivers/video/tegra/nvmap/nvmap_ioctl.h
+++ b/drivers/video/tegra/nvmap/nvmap_ioctl.h
@@ -31,6 +31,8 @@ int nvmap_ioctl_getfd(struct file *filp, void __user *arg);
 
 int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
 
+int nvmap_ioctl_dealloc(struct file *filp, unsigned long arg);
+
 int nvmap_ioctl_alloc_kind(struct file *filp, void __user *arg);
 
 int nvmap_ioctl_alloc_ivm(struct file *filp, void __user *arg);
diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h
index 781260b1d..9d668cd2e 100644
--- a/drivers/video/tegra/nvmap/nvmap_priv.h
+++ b/drivers/video/tegra/nvmap/nvmap_priv.h
@@ -174,6 +174,7 @@ struct nvmap_handle {
         bool from_va;           /* handle memory is from VA */
         u32 heap_type;          /* handle heap is allocated from */
         u32 userflags;          /* flags passed from userspace */
+        u32 userheap_mask;      /* heap_mask passed from userspace */
         void *vaddr;            /* mapping used inside kernel */
         struct list_head vmas;  /* list of all user vma's */
         atomic_t umap_count;    /* number of outstanding maps from user */
@@ -191,6 +192,9 @@ struct nvmap_handle_info {
         struct nvmap_handle *handle;
         struct list_head maps;
         struct mutex maps_lock;
+        // Allow drivers such as nvgpu to optionally store pointers to their
+        // own internal state tracking structures
+        struct list_head user_priv_list;
 };
 
 struct nvmap_tag_entry {
@@ -415,12 +419,16 @@ void nvmap_free_handle(struct nvmap_client *c, struct nvmap_handle *h);
 
 void nvmap_free_handle_fd(struct nvmap_client *c, int fd);
 
+void nvmap_dealloc_fd(struct nvmap_client *client, int fd);
+
 int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h);
 
 void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h);
 
 int is_nvmap_vma(struct vm_area_struct *vma);
 
+bool dmabuf_is_nvmap(struct dma_buf *dmabuf);
+
 int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h);
 struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd(
                                 struct nvmap_client *client, int fd);
diff --git a/include/linux/nvmap.h b/include/linux/nvmap.h
index b8d267691..7a8181ca5 100644
--- a/include/linux/nvmap.h
+++ b/include/linux/nvmap.h
@@ -65,6 +65,24 @@ ulong nvmap_iovmm_get_used_pages(void);
 int nvmap_register_vidmem_carveout(struct device *dma_dev,
                 phys_addr_t base, size_t size);
 
+/* The following two functions are to help enable DRAM
+ * overprovising via paging. These functions allow the
+ * physical frames backing an nvmap dma_buf to be freed
+ * (for example, after their contents has been saved
+ * elsewhere by paging logic), and to be reallocated
+ * (such as when other code is ready to repopulate them).
+ * Both functions preserve any open nvmap handles.
+ */
+int nvmap_dealloc_dmabuf(struct dma_buf *dmabuf);
+int nvmap_realloc_dmabuf(struct dma_buf *dmabuf);
+
+/* Some drivers (such as nvgpu) store parallel structures
+ * for each dmabuf to track internal state. To allow these
+ * drivers to quickly access their state from a *dmabuf or
+ * FD, we allow them access to a per-dmabuf list_head.
+ */
+struct list_head* nvmap_get_priv_list(struct dma_buf *dmabuf);
+
 /*
  * A heap can be mapped to memory other than DRAM.
  * The HW, controls the memory, can be power gated/ungated
diff --git a/include/uapi/linux/nvmap.h b/include/uapi/linux/nvmap.h
index 66eda75ce..5fea19ac8 100644
--- a/include/uapi/linux/nvmap.h
+++ b/include/uapi/linux/nvmap.h
@@ -304,6 +304,8 @@ struct nvmap_handle_parameters {
  */
 #define NVMAP_IOC_FREE       _IO(NVMAP_IOC_MAGIC, 4)
 
+#define NVMAP_IOC_DEALLOC    _IO(NVMAP_IOC_MAGIC, 9)
+
 /* Maps the region of the specified handle into a user-provided virtual address
  * that was previously created via an mmap syscall on this fd */
 #define NVMAP_IOC_MMAP       _IOWR(NVMAP_IOC_MAGIC, 5, struct nvmap_map_caller)