From b3446bc0b6fca6cb992667f80a95f8503b6a652a Mon Sep 17 00:00:00 2001 From: Alex Waterman Date: Thu, 7 Sep 2017 15:27:55 -0700 Subject: gpu: nvgpu: Move dma_buf usage from mm_gk20a.c Move most of the dma_buf usage present in the mm_gk20a.c code out to Linux specific code and some commom/mm code. There's two primary groups of code: 1. dma_buf priv field code (for holding comptag data) 2. Comptag usage that relies on dma_buf pointers For (1) the dma_buf code was simply moved to common/linux/dmabuf.c since most of this code is clearly Linux specific. The comptag code was a bit more complicated since there is two parts to the comptag code. Firstly there's the code that manages the comptag memory. This is essentially a simple allocator. This was moved to common/mm/comptags.c since it can be shared across all chips. The second set of code is moved to common/linux/comptags.c since it is the interface between dma_bufs and the comptag memory. Two other fixes were done as well: - Add struct gk20a to the comptag allocator init so that the proper nvgpu_vzalloc() function could be used. - Add necessary includes to common/linux/vm_priv.h. JIRA NVGPU-30 JIRA NVGPU-138 Change-Id: I96c57f2763e5ebe18a2f2ee4b33e0e1a2597848c Signed-off-by: Alex Waterman Reviewed-on: https://git-master.nvidia.com/r/1566628 Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/gpu/nvgpu/Makefile.nvgpu | 3 + drivers/gpu/nvgpu/common/linux/cde.c | 1 + drivers/gpu/nvgpu/common/linux/comptags.c | 70 +++++++ drivers/gpu/nvgpu/common/linux/dmabuf.c | 247 ++++++++++++++++++++++++ drivers/gpu/nvgpu/common/linux/dmabuf.h | 73 +++++++ drivers/gpu/nvgpu/common/linux/vm.c | 29 +-- drivers/gpu/nvgpu/common/linux/vm_priv.h | 5 - drivers/gpu/nvgpu/common/mm/comptags.c | 95 +++++++++ drivers/gpu/nvgpu/gk20a/gr_gk20a.c | 32 +--- drivers/gpu/nvgpu/gk20a/gr_gk20a.h | 14 +- drivers/gpu/nvgpu/gk20a/mm_gk20a.c | 298 +---------------------------- drivers/gpu/nvgpu/gk20a/mm_gk20a.h | 19 -- drivers/gpu/nvgpu/gm20b/ltc_gm20b.c | 2 +- drivers/gpu/nvgpu/gp10b/ltc_gp10b.c | 2 +- drivers/gpu/nvgpu/include/nvgpu/comptags.h | 57 ++++++ drivers/gpu/nvgpu/vgpu/gr_vgpu.c | 2 +- drivers/gpu/nvgpu/vgpu/ltc_vgpu.c | 2 +- 17 files changed, 556 insertions(+), 395 deletions(-) create mode 100644 drivers/gpu/nvgpu/common/linux/comptags.c create mode 100644 drivers/gpu/nvgpu/common/linux/dmabuf.c create mode 100644 drivers/gpu/nvgpu/common/linux/dmabuf.h create mode 100644 drivers/gpu/nvgpu/common/mm/comptags.c create mode 100644 drivers/gpu/nvgpu/include/nvgpu/comptags.h diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu index 42447e0c..81b0d1b8 100644 --- a/drivers/gpu/nvgpu/Makefile.nvgpu +++ b/drivers/gpu/nvgpu/Makefile.nvgpu @@ -51,6 +51,8 @@ nvgpu-y := \ common/linux/rwsem.o \ common/linux/cde_gm20b.o \ common/linux/cde_gp10b.o \ + common/linux/comptags.o \ + common/linux/dmabuf.o \ common/mm/nvgpu_allocator.o \ common/mm/bitmap_allocator.o \ common/mm/buddy_allocator.o \ @@ -61,6 +63,7 @@ nvgpu-y := \ common/mm/vm.o \ common/mm/vm_area.o \ common/mm/nvgpu_mem.o \ + common/mm/comptags.o \ common/bus.o \ common/enabled.o \ common/pramin.o \ diff --git a/drivers/gpu/nvgpu/common/linux/cde.c b/drivers/gpu/nvgpu/common/linux/cde.c index c3a9b770..577d86e8 100644 --- a/drivers/gpu/nvgpu/common/linux/cde.c +++ b/drivers/gpu/nvgpu/common/linux/cde.c @@ -39,6 +39,7 @@ #include "cde.h" #include "os_linux.h" +#include "dmabuf.h" #include #include diff --git a/drivers/gpu/nvgpu/common/linux/comptags.c b/drivers/gpu/nvgpu/common/linux/comptags.c new file mode 100644 index 00000000..517429d8 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/comptags.c @@ -0,0 +1,70 @@ +/* +* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include + +#include + +#include "dmabuf.h" + +void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf, + struct gk20a_comptags *comptags) +{ + struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); + + if (!comptags) + return; + + if (!priv) { + memset(comptags, 0, sizeof(*comptags)); + return; + } + + *comptags = priv->comptags; +} + +int gk20a_alloc_comptags(struct gk20a *g, + struct device *dev, + struct dma_buf *dmabuf, + struct gk20a_comptag_allocator *allocator, + u32 lines) +{ + struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); + u32 ctaglines_allocsize; + u32 offset; + int err; + + if (!priv) + return -ENOSYS; + + if (!lines) + return -EINVAL; + + ctaglines_allocsize = lines; + + /* store the allocator so we can use it when we free the ctags */ + priv->comptag_allocator = allocator; + err = gk20a_comptaglines_alloc(allocator, &offset, + ctaglines_allocsize); + if (err) + return err; + + priv->comptags.offset = offset; + priv->comptags.lines = lines; + priv->comptags.allocated_lines = ctaglines_allocsize; + + return 0; +} diff --git a/drivers/gpu/nvgpu/common/linux/dmabuf.c b/drivers/gpu/nvgpu/common/linux/dmabuf.c new file mode 100644 index 00000000..0b07b255 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/dmabuf.c @@ -0,0 +1,247 @@ +/* +* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include + +#include +#include + +#include + +#include "gk20a/gk20a.h" +#include "gk20a/platform_gk20a.h" + +#include "dmabuf.h" +#include "vm_priv.h" +#include "os_linux.h" + +static void gk20a_mm_delete_priv(void *_priv) +{ + struct gk20a_buffer_state *s, *s_tmp; + struct gk20a_dmabuf_priv *priv = _priv; + struct gk20a *g; + + if (!priv) + return; + + g = priv->g; + + if (priv->comptags.lines) { + BUG_ON(!priv->comptag_allocator); + gk20a_comptaglines_free(priv->comptag_allocator, + priv->comptags.offset, + priv->comptags.allocated_lines); + } + + /* Free buffer states */ + nvgpu_list_for_each_entry_safe(s, s_tmp, &priv->states, + gk20a_buffer_state, list) { + gk20a_fence_put(s->fence); + nvgpu_list_del(&s->list); + nvgpu_kfree(g, s); + } + + nvgpu_kfree(g, priv); +} + +enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, + struct dma_buf *dmabuf) +{ + struct gk20a *buf_owner = nvgpu_vidmem_buf_owner(dmabuf); + bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY); + + if (buf_owner == NULL) { + /* Not nvgpu-allocated, assume system memory */ + return APERTURE_SYSMEM; + } else if (WARN_ON(buf_owner == g && unified_memory)) { + /* Looks like our video memory, but this gpu doesn't support + * it. Warn about a bug and bail out */ + nvgpu_warn(g, + "dmabuf is our vidmem but we don't have local vidmem"); + return APERTURE_INVALID; + } else if (buf_owner != g) { + /* Someone else's vidmem */ + return APERTURE_INVALID; + } else { + /* Yay, buf_owner == g */ + return APERTURE_VIDMEM; + } +} + +struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf) +{ + struct gk20a_dmabuf_priv *priv; + + priv = dma_buf_get_drvdata(dmabuf, dev); + if (WARN_ON(!priv)) + return ERR_PTR(-EINVAL); + + nvgpu_mutex_acquire(&priv->lock); + + if (priv->pin_count == 0) { + priv->attach = dma_buf_attach(dmabuf, dev); + if (IS_ERR(priv->attach)) { + nvgpu_mutex_release(&priv->lock); + return (struct sg_table *)priv->attach; + } + + priv->sgt = dma_buf_map_attachment(priv->attach, + DMA_BIDIRECTIONAL); + if (IS_ERR(priv->sgt)) { + dma_buf_detach(dmabuf, priv->attach); + nvgpu_mutex_release(&priv->lock); + return priv->sgt; + } + } + + priv->pin_count++; + nvgpu_mutex_release(&priv->lock); + return priv->sgt; +} + +void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf, + struct sg_table *sgt) +{ + struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); + dma_addr_t dma_addr; + + if (IS_ERR(priv) || !priv) + return; + + nvgpu_mutex_acquire(&priv->lock); + WARN_ON(priv->sgt != sgt); + priv->pin_count--; + WARN_ON(priv->pin_count < 0); + dma_addr = sg_dma_address(priv->sgt->sgl); + if (priv->pin_count == 0) { + dma_buf_unmap_attachment(priv->attach, priv->sgt, + DMA_BIDIRECTIONAL); + dma_buf_detach(dmabuf, priv->attach); + } + nvgpu_mutex_release(&priv->lock); +} + +int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev) +{ + struct gk20a *g = gk20a_get_platform(dev)->g; + struct gk20a_dmabuf_priv *priv; + static u64 priv_count = 0; + + priv = dma_buf_get_drvdata(dmabuf, dev); + if (likely(priv)) + return 0; + + nvgpu_mutex_acquire(&g->mm.priv_lock); + priv = dma_buf_get_drvdata(dmabuf, dev); + if (priv) + goto priv_exist_or_err; + + priv = nvgpu_kzalloc(g, sizeof(*priv)); + if (!priv) { + priv = ERR_PTR(-ENOMEM); + goto priv_exist_or_err; + } + + nvgpu_mutex_init(&priv->lock); + nvgpu_init_list_node(&priv->states); + priv->buffer_id = ++priv_count; + priv->g = g; + dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv); + +priv_exist_or_err: + nvgpu_mutex_release(&g->mm.priv_lock); + if (IS_ERR(priv)) + return -ENOMEM; + + return 0; +} + +int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g, + u64 offset, struct gk20a_buffer_state **state) +{ + int err = 0; + struct gk20a_dmabuf_priv *priv; + struct gk20a_buffer_state *s; + struct device *dev = dev_from_gk20a(g); + + if (WARN_ON(offset >= (u64)dmabuf->size)) + return -EINVAL; + + err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev); + if (err) + return err; + + priv = dma_buf_get_drvdata(dmabuf, dev); + if (WARN_ON(!priv)) + return -ENOSYS; + + nvgpu_mutex_acquire(&priv->lock); + + nvgpu_list_for_each_entry(s, &priv->states, gk20a_buffer_state, list) + if (s->offset == offset) + goto out; + + /* State not found, create state. */ + s = nvgpu_kzalloc(g, sizeof(*s)); + if (!s) { + err = -ENOMEM; + goto out; + } + + s->offset = offset; + nvgpu_init_list_node(&s->list); + nvgpu_mutex_init(&s->lock); + nvgpu_list_add_tail(&s->list, &priv->states); + +out: + nvgpu_mutex_release(&priv->lock); + if (!err) + *state = s; + return err; +} + +int gk20a_mm_get_buffer_info(struct device *dev, int dmabuf_fd, + u64 *buffer_id, u64 *buffer_len) +{ + struct dma_buf *dmabuf; + struct gk20a_dmabuf_priv *priv; + int err = 0; + + dmabuf = dma_buf_get(dmabuf_fd); + if (IS_ERR(dmabuf)) { + dev_warn(dev, "%s: fd %d is not a dmabuf", __func__, dmabuf_fd); + return PTR_ERR(dmabuf); + } + + err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev); + if (err) { + dev_warn(dev, "Failed to allocate dmabuf drvdata (err = %d)", + err); + goto clean_up; + } + + priv = dma_buf_get_drvdata(dmabuf, dev); + if (likely(priv)) { + *buffer_id = priv->buffer_id; + *buffer_len = dmabuf->size; + } + +clean_up: + dma_buf_put(dmabuf); + return err; +} diff --git a/drivers/gpu/nvgpu/common/linux/dmabuf.h b/drivers/gpu/nvgpu/common/linux/dmabuf.h new file mode 100644 index 00000000..718386c5 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/dmabuf.h @@ -0,0 +1,73 @@ +/* +* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __COMMON_LINUX_DMABUF_H__ +#define __COMMON_LINUX_DMABUF_H__ + +#include +#include +#include +#include + +struct sg_table; +struct dma_buf; +struct dma_buf_attachment; +struct device; + +struct gk20a; +struct gk20a_buffer_state; + +struct gk20a_dmabuf_priv { + struct nvgpu_mutex lock; + + struct gk20a *g; + + struct gk20a_comptag_allocator *comptag_allocator; + struct gk20a_comptags comptags; + + struct dma_buf_attachment *attach; + struct sg_table *sgt; + + int pin_count; + + struct nvgpu_list_node states; + + u64 buffer_id; +}; + +/* + * These are implemented in common/linux/comptags.c - these are dmabuf related + * functions though so they are defined here. They cannot be defined in + * since that file must be OS agnostic. + */ +int gk20a_alloc_comptags(struct gk20a *g, + struct device *dev, + struct dma_buf *dmabuf, + struct gk20a_comptag_allocator *allocator, + u32 lines); +void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf, + struct gk20a_comptags *comptags); + +struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf); +void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf, + struct sg_table *sgt); + +int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev); + +int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g, + u64 offset, struct gk20a_buffer_state **state); + +#endif diff --git a/drivers/gpu/nvgpu/common/linux/vm.c b/drivers/gpu/nvgpu/common/linux/vm.c index 45058321..b686d616 100644 --- a/drivers/gpu/nvgpu/common/linux/vm.c +++ b/drivers/gpu/nvgpu/common/linux/vm.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include @@ -36,33 +35,7 @@ #include "vm_priv.h" #include "os_linux.h" - -/* - * Temporary location for this code until a dmabuf.c file exists. - */ -enum nvgpu_aperture gk20a_dmabuf_aperture(struct gk20a *g, - struct dma_buf *dmabuf) -{ - struct gk20a *buf_owner = nvgpu_vidmem_buf_owner(dmabuf); - bool unified_memory = nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY); - - if (buf_owner == NULL) { - /* Not nvgpu-allocated, assume system memory */ - return APERTURE_SYSMEM; - } else if (WARN_ON(buf_owner == g && unified_memory)) { - /* Looks like our video memory, but this gpu doesn't support - * it. Warn about a bug and bail out */ - nvgpu_warn(g, - "dmabuf is our vidmem but we don't have local vidmem"); - return APERTURE_INVALID; - } else if (buf_owner != g) { - /* Someone else's vidmem */ - return APERTURE_INVALID; - } else { - /* Yay, buf_owner == g */ - return APERTURE_VIDMEM; - } -} +#include "dmabuf.h" static struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_reverse( struct vm_gk20a *vm, struct dma_buf *dmabuf, u32 kind) diff --git a/drivers/gpu/nvgpu/common/linux/vm_priv.h b/drivers/gpu/nvgpu/common/linux/vm_priv.h index 1eadf1d0..fa173d59 100644 --- a/drivers/gpu/nvgpu/common/linux/vm_priv.h +++ b/drivers/gpu/nvgpu/common/linux/vm_priv.h @@ -102,10 +102,5 @@ int setup_buffer_kind_and_compression(struct vm_gk20a *vm, u32 flags, struct buffer_attrs *bfr, enum gmmu_pgsz_gk20a pgsz_idx); -int gk20a_alloc_comptags(struct gk20a *g, - struct device *dev, - struct dma_buf *dmabuf, - struct gk20a_comptag_allocator *allocator, - u32 lines); #endif diff --git a/drivers/gpu/nvgpu/common/mm/comptags.c b/drivers/gpu/nvgpu/common/mm/comptags.c new file mode 100644 index 00000000..01ab646a --- /dev/null +++ b/drivers/gpu/nvgpu/common/mm/comptags.c @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include "gk20a/gk20a.h" + +int gk20a_comptaglines_alloc(struct gk20a_comptag_allocator *allocator, + u32 *offset, u32 len) +{ + unsigned long addr; + int err = 0; + + nvgpu_mutex_acquire(&allocator->lock); + addr = bitmap_find_next_zero_area(allocator->bitmap, allocator->size, + 0, len, 0); + if (addr < allocator->size) { + /* number zero is reserved; bitmap base is 1 */ + *offset = 1 + addr; + bitmap_set(allocator->bitmap, addr, len); + } else { + err = -ENOMEM; + } + nvgpu_mutex_release(&allocator->lock); + + return err; +} + +void gk20a_comptaglines_free(struct gk20a_comptag_allocator *allocator, + u32 offset, u32 len) +{ + /* number zero is reserved; bitmap base is 1 */ + u32 addr = offset - 1; + + WARN_ON(offset == 0); + WARN_ON(addr > allocator->size); + WARN_ON(addr + len > allocator->size); + + nvgpu_mutex_acquire(&allocator->lock); + bitmap_clear(allocator->bitmap, addr, len); + nvgpu_mutex_release(&allocator->lock); +} + +int gk20a_comptag_allocator_init(struct gk20a *g, + struct gk20a_comptag_allocator *allocator, + unsigned long size) +{ + nvgpu_mutex_init(&allocator->lock); + + /* + * 0th comptag is special and is never used. The base for this bitmap + * is 1, and its size is one less than the size of comptag store. + */ + size--; + allocator->bitmap = nvgpu_vzalloc(g, + BITS_TO_LONGS(size) * sizeof(long)); + if (!allocator->bitmap) + return -ENOMEM; + + allocator->size = size; + + return 0; +} + +void gk20a_comptag_allocator_destroy(struct gk20a *g, + struct gk20a_comptag_allocator *allocator) +{ + /* + * called only when exiting the driver (gk20a_remove, or unwinding the + * init stage); no users should be active, so taking the mutex is + * unnecessary here. + */ + allocator->size = 0; + nvgpu_vfree(allocator->g, allocator->bitmap); +} diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c index 57c1c0bc..700dcdf8 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c @@ -3044,36 +3044,6 @@ out: return err; } -int gk20a_comptag_allocator_init(struct gk20a_comptag_allocator *allocator, - unsigned long size) -{ - nvgpu_mutex_init(&allocator->lock); - /* - * 0th comptag is special and is never used. The base for this bitmap - * is 1, and its size is one less than the size of comptag store. - */ - size--; - allocator->bitmap = vzalloc(BITS_TO_LONGS(size) * sizeof(long)); - if (!allocator->bitmap) - return -ENOMEM; - allocator->size = size; - return 0; -} - -void gk20a_comptag_allocator_destroy(struct gk20a_comptag_allocator *allocator) -{ - struct gr_gk20a *gr = container_of(allocator, - struct gr_gk20a, comp_tags); - - /* - * called only when exiting the driver (gk20a_remove, or unwinding the - * init stage); no users should be active, so taking the mutex is - * unnecessary here. - */ - allocator->size = 0; - nvgpu_vfree(gr->g, allocator->bitmap); -} - static void gk20a_remove_gr_support(struct gr_gk20a *gr) { struct gk20a *g = gr->g; @@ -3148,7 +3118,7 @@ static void gk20a_remove_gr_support(struct gr_gk20a *gr) nvgpu_big_free(g, gr->ctx_vars.hwpm_ctxsw_buffer_offset_map); gr->ctx_vars.hwpm_ctxsw_buffer_offset_map = NULL; - gk20a_comptag_allocator_destroy(&gr->comp_tags); + gk20a_comptag_allocator_destroy(g, &gr->comp_tags); } static int gr_gk20a_init_gr_config(struct gk20a *g, struct gr_gk20a *gr) diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h index 5fab43ca..84eb8970 100644 --- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.h @@ -36,6 +36,8 @@ #include "gr_ctx_gk20a.h" #include "mm_gk20a.h" +#include + #define GR_IDLE_CHECK_DEFAULT 10 /* usec */ #define GR_IDLE_CHECK_MAX 200 /* usec */ #define GR_FECS_POLL_INTERVAL 5 /* usec */ @@ -342,13 +344,7 @@ struct gr_gk20a { u32 max_comptag_mem; /* max memory size (MB) for comptag */ struct compbit_store_desc compbit_store; - struct gk20a_comptag_allocator { - struct nvgpu_mutex lock; - /* this bitmap starts at ctag 1. 0th cannot be taken */ - unsigned long *bitmap; - /* size of bitmap, not max ctags, so one less */ - unsigned long size; - } comp_tags; + struct gk20a_comptag_allocator comp_tags; struct gr_zcull_gk20a zcull; @@ -503,10 +499,6 @@ int gk20a_init_gr_support(struct gk20a *g); int gk20a_enable_gr_hw(struct gk20a *g); int gk20a_gr_reset(struct gk20a *g); void gk20a_gr_wait_initialized(struct gk20a *g); -/* real size here, but first (ctag 0) isn't used */ -int gk20a_comptag_allocator_init(struct gk20a_comptag_allocator *allocator, - unsigned long size); -void gk20a_comptag_allocator_destroy(struct gk20a_comptag_allocator *allocator); int gk20a_init_gr_channel(struct channel_gk20a *ch_gk20a); diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c index 8936cd03..69d9e983 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c @@ -22,7 +22,6 @@ * DEALINGS IN THE SOFTWARE. */ -#include #include #include #include @@ -70,6 +69,7 @@ * all the common APIs no longers have Linux stuff in them. */ #include "common/linux/vm_priv.h" +#include "common/linux/dmabuf.h" /* * GPU mapping life cycle @@ -108,190 +108,6 @@ static int __must_check gk20a_init_hwpm(struct mm_gk20a *mm); static int __must_check gk20a_init_cde_vm(struct mm_gk20a *mm); static int __must_check gk20a_init_ce_vm(struct mm_gk20a *mm); -struct gk20a_dmabuf_priv { - struct nvgpu_mutex lock; - - struct gk20a *g; - - struct gk20a_comptag_allocator *comptag_allocator; - struct gk20a_comptags comptags; - - struct dma_buf_attachment *attach; - struct sg_table *sgt; - - int pin_count; - - struct nvgpu_list_node states; - - u64 buffer_id; -}; - -static int gk20a_comptaglines_alloc(struct gk20a_comptag_allocator *allocator, - u32 *offset, u32 len) -{ - unsigned long addr; - int err = 0; - - nvgpu_mutex_acquire(&allocator->lock); - addr = bitmap_find_next_zero_area(allocator->bitmap, allocator->size, - 0, len, 0); - if (addr < allocator->size) { - /* number zero is reserved; bitmap base is 1 */ - *offset = 1 + addr; - bitmap_set(allocator->bitmap, addr, len); - } else { - err = -ENOMEM; - } - nvgpu_mutex_release(&allocator->lock); - - return err; -} - -static void gk20a_comptaglines_free(struct gk20a_comptag_allocator *allocator, - u32 offset, u32 len) -{ - /* number zero is reserved; bitmap base is 1 */ - u32 addr = offset - 1; - WARN_ON(offset == 0); - WARN_ON(addr > allocator->size); - WARN_ON(addr + len > allocator->size); - - nvgpu_mutex_acquire(&allocator->lock); - bitmap_clear(allocator->bitmap, addr, len); - nvgpu_mutex_release(&allocator->lock); -} - -static void gk20a_mm_delete_priv(void *_priv) -{ - struct gk20a_buffer_state *s, *s_tmp; - struct gk20a_dmabuf_priv *priv = _priv; - struct gk20a *g; - - if (!priv) - return; - - g = priv->g; - - if (priv->comptags.lines) { - BUG_ON(!priv->comptag_allocator); - gk20a_comptaglines_free(priv->comptag_allocator, - priv->comptags.offset, - priv->comptags.allocated_lines); - } - - /* Free buffer states */ - nvgpu_list_for_each_entry_safe(s, s_tmp, &priv->states, - gk20a_buffer_state, list) { - gk20a_fence_put(s->fence); - nvgpu_list_del(&s->list); - nvgpu_kfree(g, s); - } - - nvgpu_kfree(g, priv); -} - -struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf) -{ - struct gk20a_dmabuf_priv *priv; - - priv = dma_buf_get_drvdata(dmabuf, dev); - if (WARN_ON(!priv)) - return ERR_PTR(-EINVAL); - - nvgpu_mutex_acquire(&priv->lock); - - if (priv->pin_count == 0) { - priv->attach = dma_buf_attach(dmabuf, dev); - if (IS_ERR(priv->attach)) { - nvgpu_mutex_release(&priv->lock); - return (struct sg_table *)priv->attach; - } - - priv->sgt = dma_buf_map_attachment(priv->attach, - DMA_BIDIRECTIONAL); - if (IS_ERR(priv->sgt)) { - dma_buf_detach(dmabuf, priv->attach); - nvgpu_mutex_release(&priv->lock); - return priv->sgt; - } - } - - priv->pin_count++; - nvgpu_mutex_release(&priv->lock); - return priv->sgt; -} - -void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf, - struct sg_table *sgt) -{ - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); - dma_addr_t dma_addr; - - if (IS_ERR(priv) || !priv) - return; - - nvgpu_mutex_acquire(&priv->lock); - WARN_ON(priv->sgt != sgt); - priv->pin_count--; - WARN_ON(priv->pin_count < 0); - dma_addr = sg_dma_address(priv->sgt->sgl); - if (priv->pin_count == 0) { - dma_buf_unmap_attachment(priv->attach, priv->sgt, - DMA_BIDIRECTIONAL); - dma_buf_detach(dmabuf, priv->attach); - } - nvgpu_mutex_release(&priv->lock); -} - -void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf, - struct gk20a_comptags *comptags) -{ - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); - - if (!comptags) - return; - - if (!priv) { - memset(comptags, 0, sizeof(*comptags)); - return; - } - - *comptags = priv->comptags; -} - -int gk20a_alloc_comptags(struct gk20a *g, - struct device *dev, - struct dma_buf *dmabuf, - struct gk20a_comptag_allocator *allocator, - u32 lines) -{ - struct gk20a_dmabuf_priv *priv = dma_buf_get_drvdata(dmabuf, dev); - u32 ctaglines_allocsize; - u32 offset; - int err; - - if (!priv) - return -ENOSYS; - - if (!lines) - return -EINVAL; - - ctaglines_allocsize = lines; - - /* store the allocator so we can use it when we free the ctags */ - priv->comptag_allocator = allocator; - err = gk20a_comptaglines_alloc(allocator, &offset, - ctaglines_allocsize); - if (err) - return err; - - priv->comptags.offset = offset; - priv->comptags.lines = lines; - priv->comptags.allocated_lines = ctaglines_allocsize; - - return 0; -} - static int gk20a_init_mm_reset_enable_hw(struct gk20a *g) { gk20a_dbg_fn(""); @@ -1037,87 +853,6 @@ int gk20a_vm_bind_channel(struct gk20a_as_share *as_share, return __gk20a_vm_bind_channel(as_share->vm, ch); } -int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev) -{ - struct gk20a *g = gk20a_get_platform(dev)->g; - struct gk20a_dmabuf_priv *priv; - static u64 priv_count = 0; - - priv = dma_buf_get_drvdata(dmabuf, dev); - if (likely(priv)) - return 0; - - nvgpu_mutex_acquire(&g->mm.priv_lock); - priv = dma_buf_get_drvdata(dmabuf, dev); - if (priv) - goto priv_exist_or_err; - - priv = nvgpu_kzalloc(g, sizeof(*priv)); - if (!priv) { - priv = ERR_PTR(-ENOMEM); - goto priv_exist_or_err; - } - - nvgpu_mutex_init(&priv->lock); - nvgpu_init_list_node(&priv->states); - priv->buffer_id = ++priv_count; - priv->g = g; - dma_buf_set_drvdata(dmabuf, dev, priv, gk20a_mm_delete_priv); - -priv_exist_or_err: - nvgpu_mutex_release(&g->mm.priv_lock); - if (IS_ERR(priv)) - return -ENOMEM; - - return 0; -} - -int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g, - u64 offset, struct gk20a_buffer_state **state) -{ - int err = 0; - struct gk20a_dmabuf_priv *priv; - struct gk20a_buffer_state *s; - struct device *dev = dev_from_gk20a(g); - - if (WARN_ON(offset >= (u64)dmabuf->size)) - return -EINVAL; - - err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev); - if (err) - return err; - - priv = dma_buf_get_drvdata(dmabuf, dev); - if (WARN_ON(!priv)) - return -ENOSYS; - - nvgpu_mutex_acquire(&priv->lock); - - nvgpu_list_for_each_entry(s, &priv->states, gk20a_buffer_state, list) - if (s->offset == offset) - goto out; - - /* State not found, create state. */ - s = nvgpu_kzalloc(g, sizeof(*s)); - if (!s) { - err = -ENOMEM; - goto out; - } - - s->offset = offset; - nvgpu_init_list_node(&s->list); - nvgpu_mutex_init(&s->lock); - nvgpu_list_add_tail(&s->list, &priv->states); - -out: - nvgpu_mutex_release(&priv->lock); - if (!err) - *state = s; - return err; - - -} - int nvgpu_vm_map_buffer(struct vm_gk20a *vm, int dmabuf_fd, u64 *offset_align, @@ -1613,34 +1348,3 @@ const struct gk20a_mmu_level *gk20a_mm_get_mmu_levels(struct gk20a *g, return (big_page_size == SZ_64K) ? gk20a_mm_levels_64k : gk20a_mm_levels_128k; } - -int gk20a_mm_get_buffer_info(struct device *dev, int dmabuf_fd, - u64 *buffer_id, u64 *buffer_len) -{ - struct dma_buf *dmabuf; - struct gk20a_dmabuf_priv *priv; - int err = 0; - - dmabuf = dma_buf_get(dmabuf_fd); - if (IS_ERR(dmabuf)) { - dev_warn(dev, "%s: fd %d is not a dmabuf", __func__, dmabuf_fd); - return PTR_ERR(dmabuf); - } - - err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev); - if (err) { - dev_warn(dev, "Failed to allocate dmabuf drvdata (err = %d)", - err); - goto clean_up; - } - - priv = dma_buf_get_drvdata(dmabuf, dev); - if (likely(priv)) { - *buffer_id = priv->buffer_id; - *buffer_len = dmabuf->size; - } - -clean_up: - dma_buf_put(dmabuf); - return err; -} diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h index 448496f5..04034d84 100644 --- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h @@ -117,13 +117,6 @@ gk20a_buffer_state_from_list(struct nvgpu_list_node *node) ((uintptr_t)node - offsetof(struct gk20a_buffer_state, list)); }; -struct gk20a_comptags { - u32 offset; - u32 lines; - u32 allocated_lines; - bool user_mappable; -}; - struct priv_cmd_queue { struct nvgpu_mem mem; u32 size; /* num of entries in words */ @@ -369,10 +362,6 @@ void gk20a_locked_gmmu_unmap(struct vm_gk20a *vm, bool sparse, struct vm_gk20a_mapping_batch *batch); -struct sg_table *gk20a_mm_pin(struct device *dev, struct dma_buf *dmabuf); -void gk20a_mm_unpin(struct device *dev, struct dma_buf *dmabuf, - struct sg_table *sgt); - /* vm-as interface */ struct nvgpu_as_alloc_space_args; struct nvgpu_as_free_space_args; @@ -381,14 +370,6 @@ int gk20a_vm_bind_channel(struct gk20a_as_share *as_share, struct channel_gk20a *ch); int __gk20a_vm_bind_channel(struct vm_gk20a *vm, struct channel_gk20a *ch); -void gk20a_get_comptags(struct device *dev, struct dma_buf *dmabuf, - struct gk20a_comptags *comptags); - -int gk20a_dmabuf_alloc_drvdata(struct dma_buf *dmabuf, struct device *dev); - -int gk20a_dmabuf_get_state(struct dma_buf *dmabuf, struct gk20a *g, - u64 offset, struct gk20a_buffer_state **state); - void pde_range_from_vaddr_range(struct vm_gk20a *vm, u64 addr_lo, u64 addr_hi, u32 *pde_lo, u32 *pde_hi); diff --git a/drivers/gpu/nvgpu/gm20b/ltc_gm20b.c b/drivers/gpu/nvgpu/gm20b/ltc_gm20b.c index dee46aaf..84759192 100644 --- a/drivers/gpu/nvgpu/gm20b/ltc_gm20b.c +++ b/drivers/gpu/nvgpu/gm20b/ltc_gm20b.c @@ -100,7 +100,7 @@ int gm20b_ltc_init_comptags(struct gk20a *g, struct gr_gk20a *gr) if (err) return err; - err = gk20a_comptag_allocator_init(&gr->comp_tags, max_comptag_lines); + err = gk20a_comptag_allocator_init(g, &gr->comp_tags, max_comptag_lines); if (err) return err; diff --git a/drivers/gpu/nvgpu/gp10b/ltc_gp10b.c b/drivers/gpu/nvgpu/gp10b/ltc_gp10b.c index 02cab938..9d878402 100644 --- a/drivers/gpu/nvgpu/gp10b/ltc_gp10b.c +++ b/drivers/gpu/nvgpu/gp10b/ltc_gp10b.c @@ -120,7 +120,7 @@ int gp10b_ltc_init_comptags(struct gk20a *g, struct gr_gk20a *gr) if (err) return err; - err = gk20a_comptag_allocator_init(&gr->comp_tags, max_comptag_lines); + err = gk20a_comptag_allocator_init(g, &gr->comp_tags, max_comptag_lines); if (err) return err; diff --git a/drivers/gpu/nvgpu/include/nvgpu/comptags.h b/drivers/gpu/nvgpu/include/nvgpu/comptags.h new file mode 100644 index 00000000..6e3062ec --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/comptags.h @@ -0,0 +1,57 @@ +/* +* Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __NVGPU_COMPTAGS__ +#define __NVGPU_COMPTAGS__ + +#include + +struct gk20a; + +struct gk20a_comptags { + u32 offset; + u32 lines; + u32 allocated_lines; + bool user_mappable; +}; + +struct gk20a_comptag_allocator { + struct gk20a *g; + + struct nvgpu_mutex lock; + + /* This bitmap starts at ctag 1. 0th cannot be taken. */ + unsigned long *bitmap; + + /* Size of bitmap, not max ctags, so one less. */ + unsigned long size; +}; + +/* real size here, but first (ctag 0) isn't used */ +int gk20a_comptag_allocator_init(struct gk20a *g, + struct gk20a_comptag_allocator *allocator, + unsigned long size); +void gk20a_comptag_allocator_destroy(struct gk20a *g, + struct gk20a_comptag_allocator *allocator); + +int gk20a_comptaglines_alloc(struct gk20a_comptag_allocator *allocator, + u32 *offset, u32 len); +void gk20a_comptaglines_free(struct gk20a_comptag_allocator *allocator, + u32 offset, u32 len); + + + +#endif diff --git a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c index 96d21c0a..b913847b 100644 --- a/drivers/gpu/nvgpu/vgpu/gr_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/gr_vgpu.c @@ -844,7 +844,7 @@ static void vgpu_remove_gr_support(struct gr_gk20a *gr) { gk20a_dbg_fn(""); - gk20a_comptag_allocator_destroy(&gr->comp_tags); + gk20a_comptag_allocator_destroy(gr->g, &gr->comp_tags); nvgpu_kfree(gr->g, gr->sm_error_states); gr->sm_error_states = NULL; diff --git a/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c b/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c index a6848872..fb9558e2 100644 --- a/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c +++ b/drivers/gpu/nvgpu/vgpu/ltc_vgpu.c @@ -50,7 +50,7 @@ int vgpu_ltc_init_comptags(struct gk20a *g, struct gr_gk20a *gr) if (max_comptag_lines < 2) return -ENXIO; - err = gk20a_comptag_allocator_init(&gr->comp_tags, max_comptag_lines); + err = gk20a_comptag_allocator_init(g, &gr->comp_tags, max_comptag_lines); if (err) return err; -- cgit v1.2.2