From 2c076a01e5bd0949032ef81cd0e2d37bdecafba5 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Tue, 24 May 2022 21:16:30 -0400 Subject: gpu-paging: Functions to support initial working version - Buffer deallocation (+IOCTL) - Buffer reallocation - Private dmabuf user list and accessor --- drivers/video/tegra/nvmap/nvmap_alloc.c | 86 +++++++++++++++++++++++++++----- drivers/video/tegra/nvmap/nvmap_dev.c | 4 ++ drivers/video/tegra/nvmap/nvmap_dmabuf.c | 11 ++++ drivers/video/tegra/nvmap/nvmap_ioctl.c | 13 +++++ drivers/video/tegra/nvmap/nvmap_ioctl.h | 2 + drivers/video/tegra/nvmap/nvmap_priv.h | 8 +++ include/linux/nvmap.h | 18 +++++++ include/uapi/linux/nvmap.h | 2 + 8 files changed, 131 insertions(+), 13 deletions(-) diff --git a/drivers/video/tegra/nvmap/nvmap_alloc.c b/drivers/video/tegra/nvmap/nvmap_alloc.c index 74c9e4b90..ea648b8bf 100644 --- a/drivers/video/tegra/nvmap/nvmap_alloc.c +++ b/drivers/video/tegra/nvmap/nvmap_alloc.c @@ -775,6 +775,7 @@ int nvmap_alloc_handle(struct nvmap_client *client, nvmap_stats_read(NS_TOTAL), nvmap_stats_read(NS_ALLOC)); h->userflags = flags; + h->userheap_mask = heap_mask; nr_page = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT); /* Force mapping to uncached for VPR memory. */ if (heap_mask & (NVMAP_HEAP_CARVEOUT_VPR | ~nvmap_dev->cpu_access_mask)) @@ -911,20 +912,9 @@ int nvmap_alloc_handle_from_va(struct nvmap_client *client, return err; } -void _nvmap_handle_free(struct nvmap_handle *h) +void __nvmap_handle_free(struct nvmap_handle *h) { unsigned int i, nr_page, page_index = 0; - struct nvmap_handle_dmabuf_priv *curr, *next; - - list_for_each_entry_safe(curr, next, &h->dmabuf_priv, list) { - curr->priv_release(curr->priv); - list_del(&curr->list); - kzfree(curr); - } - - if (nvmap_handle_remove(nvmap_dev, h) != 0) - return; - if (!h->alloc) goto out; @@ -980,8 +970,24 @@ void _nvmap_handle_free(struct nvmap_handle *h) } nvmap_altfree(h->pgalloc.pages, nr_page * sizeof(struct page *)); - out: + h->alloc = false; +} + +void _nvmap_handle_free(struct nvmap_handle *h) +{ + struct nvmap_handle_dmabuf_priv *curr, *next; + + list_for_each_entry_safe(curr, next, &h->dmabuf_priv, list) { + curr->priv_release(curr->priv); + list_del(&curr->list); + kzfree(curr); + } + + if (nvmap_handle_remove(nvmap_dev, h) != 0) + return; + __nvmap_handle_free(h); + // Next two calls were originally in the `out` clause NVMAP_TAG_TRACE(trace_nvmap_destroy_handle, NULL, get_current()->pid, 0, NVMAP_TP_ARGS_H(h)); kfree(h); @@ -1041,3 +1047,57 @@ void nvmap_free_handle_fd(struct nvmap_client *client, nvmap_handle_put(handle); } } + +// Frees backing pages. Handle remains valid. +void nvmap_dealloc_fd(struct nvmap_client *client, int fd) +{ + struct nvmap_handle *handle = nvmap_handle_get_from_fd(fd); + if (handle) + __nvmap_handle_free(handle); +} + +// For deallocations from elsewhere in the kernel. +// (Userspace should use NVMAP_IOC_DEALLOC ioctl.) +int nvmap_dealloc_dmabuf(struct dma_buf *dmabuf) { + struct nvmap_handle_info *info; + + if (!dmabuf_is_nvmap(dmabuf)) + return -EINVAL; + info = dmabuf->priv; + // Fail if mapped into kernelspace + // (Unclear if this is necessary, but no handles have been observed with this + // set, and so testing has not been possible.) + if (atomic_read(&info->handle->kmap_count)) + goto out_busy; + // Buffer cannot currently be mapped into userspace (via mmap) + if (atomic_read(&info->handle->umap_count)) + goto out_busy; + // Other devices should not still have this pinned into DMA memory + if (atomic_read(&info->handle->pin)) + goto out_busy; + __nvmap_handle_free(info->handle); + return 0; +out_busy: + printk(KERN_WARNING "nvmap: Unmap requested on FD that is currently in-use! (%d h->pin, %d h->kmap_count, %d h->umap_count)\n", + atomic_read(&info->handle->pin), + atomic_read(&info->handle->kmap_count), + atomic_read(&info->handle->umap_count)); + return -EBUSY; +} +EXPORT_SYMBOL(nvmap_dealloc_dmabuf); + +// For reallocations from elsewhere in the kernel. +// (Userspace should use NVMAP_IOC_ALLOC ioctl.) +int nvmap_realloc_dmabuf(struct dma_buf *dmabuf) { + struct nvmap_handle_info *info; + struct nvmap_handle *h; + + if (!dmabuf_is_nvmap(dmabuf)) + return -EINVAL; + info = dmabuf->priv; + h = info->handle; + return nvmap_alloc_handle(h->owner, h, h->userheap_mask, + h->align, 0, h->userflags, + NVMAP_IVM_INVALID_PEER); +} +EXPORT_SYMBOL(nvmap_realloc_dmabuf); diff --git a/drivers/video/tegra/nvmap/nvmap_dev.c b/drivers/video/tegra/nvmap/nvmap_dev.c index eb5791464..a11e3edfb 100644 --- a/drivers/video/tegra/nvmap/nvmap_dev.c +++ b/drivers/video/tegra/nvmap/nvmap_dev.c @@ -360,6 +360,10 @@ static long nvmap_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err = nvmap_ioctl_alloc(filp, uarg); break; + case NVMAP_IOC_DEALLOC: + err = nvmap_ioctl_dealloc(filp, arg); + break; + case NVMAP_IOC_ALLOC_IVM: err = nvmap_ioctl_alloc_ivm(filp, uarg); break; diff --git a/drivers/video/tegra/nvmap/nvmap_dmabuf.c b/drivers/video/tegra/nvmap/nvmap_dmabuf.c index 850e07e06..33dcf5fe5 100644 --- a/drivers/video/tegra/nvmap/nvmap_dmabuf.c +++ b/drivers/video/tegra/nvmap/nvmap_dmabuf.c @@ -659,6 +659,7 @@ struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client, info->handle = handle; INIT_LIST_HEAD(&info->maps); mutex_init(&info->maps_lock); + INIT_LIST_HEAD(&info->user_priv_list); dmabuf = __dma_buf_export(info, handle->size, ro_buf); if (IS_ERR(dmabuf)) { @@ -791,3 +792,13 @@ error: dma_buf_put(dmabuf); return ret; } + +struct list_head* nvmap_get_priv_list(struct dma_buf *dmabuf) { + struct nvmap_handle_info *info; + + if (!dmabuf_is_nvmap(dmabuf)) + return ERR_PTR(-EINVAL); + info = dmabuf->priv; + return &info->user_priv_list; +} +EXPORT_SYMBOL(nvmap_get_priv_list); diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.c b/drivers/video/tegra/nvmap/nvmap_ioctl.c index 7fd13e625..93a418eb3 100644 --- a/drivers/video/tegra/nvmap/nvmap_ioctl.c +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.c @@ -460,6 +460,19 @@ int nvmap_ioctl_free(struct file *filp, unsigned long arg) return sys_close(arg); } +// Returns memory freed by deallocation +int nvmap_ioctl_dealloc(struct file *filp, unsigned long arg) +{ + struct nvmap_client *client = filp->private_data; + u64 before = nvmap_stats_read(NS_TOTAL); + + if (!arg) + return 0; + + nvmap_dealloc_fd(client, arg); + return before - nvmap_stats_read(NS_TOTAL); +} + static ssize_t rw_handle(struct nvmap_client *client, struct nvmap_handle *h, int is_read, unsigned long h_offs, unsigned long sys_addr, unsigned long h_stride, diff --git a/drivers/video/tegra/nvmap/nvmap_ioctl.h b/drivers/video/tegra/nvmap/nvmap_ioctl.h index 6c64d2abe..f1b7b9e82 100644 --- a/drivers/video/tegra/nvmap/nvmap_ioctl.h +++ b/drivers/video/tegra/nvmap/nvmap_ioctl.h @@ -31,6 +31,8 @@ int nvmap_ioctl_getfd(struct file *filp, void __user *arg); int nvmap_ioctl_alloc(struct file *filp, void __user *arg); +int nvmap_ioctl_dealloc(struct file *filp, unsigned long arg); + int nvmap_ioctl_alloc_kind(struct file *filp, void __user *arg); int nvmap_ioctl_alloc_ivm(struct file *filp, void __user *arg); diff --git a/drivers/video/tegra/nvmap/nvmap_priv.h b/drivers/video/tegra/nvmap/nvmap_priv.h index 781260b1d..9d668cd2e 100644 --- a/drivers/video/tegra/nvmap/nvmap_priv.h +++ b/drivers/video/tegra/nvmap/nvmap_priv.h @@ -174,6 +174,7 @@ struct nvmap_handle { bool from_va; /* handle memory is from VA */ u32 heap_type; /* handle heap is allocated from */ u32 userflags; /* flags passed from userspace */ + u32 userheap_mask; /* heap_mask passed from userspace */ void *vaddr; /* mapping used inside kernel */ struct list_head vmas; /* list of all user vma's */ atomic_t umap_count; /* number of outstanding maps from user */ @@ -191,6 +192,9 @@ struct nvmap_handle_info { struct nvmap_handle *handle; struct list_head maps; struct mutex maps_lock; + // Allow drivers such as nvgpu to optionally store pointers to their + // own internal state tracking structures + struct list_head user_priv_list; }; struct nvmap_tag_entry { @@ -415,12 +419,16 @@ void nvmap_free_handle(struct nvmap_client *c, struct nvmap_handle *h); void nvmap_free_handle_fd(struct nvmap_client *c, int fd); +void nvmap_dealloc_fd(struct nvmap_client *client, int fd); + int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h); void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h); int is_nvmap_vma(struct vm_area_struct *vma); +bool dmabuf_is_nvmap(struct dma_buf *dmabuf); + int nvmap_get_dmabuf_fd(struct nvmap_client *client, struct nvmap_handle *h); struct nvmap_handle *nvmap_handle_get_from_dmabuf_fd( struct nvmap_client *client, int fd); diff --git a/include/linux/nvmap.h b/include/linux/nvmap.h index b8d267691..7a8181ca5 100644 --- a/include/linux/nvmap.h +++ b/include/linux/nvmap.h @@ -65,6 +65,24 @@ ulong nvmap_iovmm_get_used_pages(void); int nvmap_register_vidmem_carveout(struct device *dma_dev, phys_addr_t base, size_t size); +/* The following two functions are to help enable DRAM + * overprovising via paging. These functions allow the + * physical frames backing an nvmap dma_buf to be freed + * (for example, after their contents has been saved + * elsewhere by paging logic), and to be reallocated + * (such as when other code is ready to repopulate them). + * Both functions preserve any open nvmap handles. + */ +int nvmap_dealloc_dmabuf(struct dma_buf *dmabuf); +int nvmap_realloc_dmabuf(struct dma_buf *dmabuf); + +/* Some drivers (such as nvgpu) store parallel structures + * for each dmabuf to track internal state. To allow these + * drivers to quickly access their state from a *dmabuf or + * FD, we allow them access to a per-dmabuf list_head. + */ +struct list_head* nvmap_get_priv_list(struct dma_buf *dmabuf); + /* * A heap can be mapped to memory other than DRAM. * The HW, controls the memory, can be power gated/ungated diff --git a/include/uapi/linux/nvmap.h b/include/uapi/linux/nvmap.h index 66eda75ce..5fea19ac8 100644 --- a/include/uapi/linux/nvmap.h +++ b/include/uapi/linux/nvmap.h @@ -304,6 +304,8 @@ struct nvmap_handle_parameters { */ #define NVMAP_IOC_FREE _IO(NVMAP_IOC_MAGIC, 4) +#define NVMAP_IOC_DEALLOC _IO(NVMAP_IOC_MAGIC, 9) + /* Maps the region of the specified handle into a user-provided virtual address * that was previously created via an mmap syscall on this fd */ #define NVMAP_IOC_MMAP _IOWR(NVMAP_IOC_MAGIC, 5, struct nvmap_map_caller) -- cgit v1.2.2