summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/gpu/nvgpu/Makefile.nvgpu1
-rw-r--r--drivers/gpu/nvgpu/common/linux/dma.c415
-rw-r--r--drivers/gpu/nvgpu/common/semaphore.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/cde_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/ce2_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/channel_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/css_gr_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/fifo_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/gr_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gk20a/ltc_common.c2
-rw-r--r--drivers/gpu/nvgpu/gk20a/mm_gk20a.c393
-rw-r--r--drivers/gpu/nvgpu/gk20a/mm_gk20a.h50
-rw-r--r--drivers/gpu/nvgpu/gk20a/pmu_gk20a.c1
-rw-r--r--drivers/gpu/nvgpu/gm20b/acr_gm20b.c1
-rw-r--r--drivers/gpu/nvgpu/gp106/acr_gp106.c1
-rw-r--r--drivers/gpu/nvgpu/gp106/gr_gp106.c2
-rw-r--r--drivers/gpu/nvgpu/gp10b/fifo_gp10b.c2
-rw-r--r--drivers/gpu/nvgpu/gp10b/gr_gp10b.c1
-rw-r--r--drivers/gpu/nvgpu/gp10b/mm_gp10b.c2
-rw-r--r--drivers/gpu/nvgpu/gp10b/rpfb_gp10b.c2
-rw-r--r--drivers/gpu/nvgpu/include/nvgpu/dma.h312
-rw-r--r--drivers/gpu/nvgpu/vgpu/fifo_vgpu.c1
-rw-r--r--drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c1
-rw-r--r--drivers/gpu/nvgpu/vgpu/mm_vgpu.c1
25 files changed, 754 insertions, 442 deletions
diff --git a/drivers/gpu/nvgpu/Makefile.nvgpu b/drivers/gpu/nvgpu/Makefile.nvgpu
index 34d8d19f..d994ac1d 100644
--- a/drivers/gpu/nvgpu/Makefile.nvgpu
+++ b/drivers/gpu/nvgpu/Makefile.nvgpu
@@ -32,6 +32,7 @@ nvgpu-y := \
32 common/linux/ioctl_tsg.o \ 32 common/linux/ioctl_tsg.o \
33 common/linux/log.o \ 33 common/linux/log.o \
34 common/linux/nvgpu_mem.o \ 34 common/linux/nvgpu_mem.o \
35 common/linux/dma.o \
35 common/mm/nvgpu_allocator.o \ 36 common/mm/nvgpu_allocator.o \
36 common/mm/bitmap_allocator.o \ 37 common/mm/bitmap_allocator.o \
37 common/mm/buddy_allocator.o \ 38 common/mm/buddy_allocator.o \
diff --git a/drivers/gpu/nvgpu/common/linux/dma.c b/drivers/gpu/nvgpu/common/linux/dma.c
new file mode 100644
index 00000000..755848ea
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/dma.c
@@ -0,0 +1,415 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <linux/dma-attrs.h>
18#include <linux/dma-mapping.h>
19
20#include <nvgpu/dma.h>
21#include <nvgpu/lock.h>
22
23#include "gk20a/gk20a.h"
24
25#if defined(CONFIG_GK20A_VIDMEM)
26static u64 __gk20a_gmmu_alloc(struct nvgpu_allocator *allocator, dma_addr_t at,
27 size_t size)
28{
29 u64 addr = 0;
30
31 if (at)
32 addr = nvgpu_alloc_fixed(allocator, at, size, 0);
33 else
34 addr = nvgpu_alloc(allocator, size);
35
36 return addr;
37}
38#endif
39
40#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
41static void gk20a_dma_flags_to_attrs(unsigned long *attrs,
42 unsigned long flags)
43#define ATTR_ARG(x) *x
44#else
45static void gk20a_dma_flags_to_attrs(struct dma_attrs *attrs,
46 unsigned long flags)
47#define ATTR_ARG(x) x
48#endif
49{
50 if (flags & NVGPU_DMA_NO_KERNEL_MAPPING)
51 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs));
52 if (flags & NVGPU_DMA_FORCE_CONTIGUOUS)
53 dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs));
54 if (flags & NVGPU_DMA_READ_ONLY)
55 dma_set_attr(DMA_ATTR_READ_ONLY, ATTR_ARG(attrs));
56#undef ATTR_ARG
57}
58
59int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem)
60{
61 return gk20a_gmmu_alloc_flags(g, 0, size, mem);
62}
63
64int gk20a_gmmu_alloc_flags(struct gk20a *g, unsigned long flags, size_t size,
65 struct nvgpu_mem *mem)
66{
67 if (g->mm.vidmem_is_vidmem) {
68 /*
69 * Force the no-kernel-mapping flag on because we don't support
70 * the lack of it for vidmem - the user should not care when
71 * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a
72 * difference, the user should use the flag explicitly anyway.
73 */
74 int err = gk20a_gmmu_alloc_flags_vid(g,
75 flags | NVGPU_DMA_NO_KERNEL_MAPPING,
76 size, mem);
77
78 if (!err)
79 return 0;
80 /*
81 * Fall back to sysmem (which may then also fail) in case
82 * vidmem is exhausted.
83 */
84 }
85
86 return gk20a_gmmu_alloc_flags_sys(g, flags, size, mem);
87}
88
89int gk20a_gmmu_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem)
90{
91 return gk20a_gmmu_alloc_flags_sys(g, 0, size, mem);
92}
93
94int gk20a_gmmu_alloc_flags_sys(struct gk20a *g, unsigned long flags,
95 size_t size, struct nvgpu_mem *mem)
96{
97 struct device *d = dev_from_gk20a(g);
98 int err;
99 dma_addr_t iova;
100
101 gk20a_dbg_fn("");
102
103 if (flags) {
104 DEFINE_DMA_ATTRS(dma_attrs);
105
106 gk20a_dma_flags_to_attrs(&dma_attrs, flags);
107
108 if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) {
109 mem->pages = dma_alloc_attrs(d,
110 size, &iova, GFP_KERNEL,
111 __DMA_ATTR(dma_attrs));
112 if (!mem->pages)
113 return -ENOMEM;
114 } else {
115 mem->cpu_va = dma_alloc_attrs(d,
116 size, &iova, GFP_KERNEL,
117 __DMA_ATTR(dma_attrs));
118 if (!mem->cpu_va)
119 return -ENOMEM;
120 }
121 } else {
122 mem->cpu_va = dma_alloc_coherent(d, size, &iova, GFP_KERNEL);
123 if (!mem->cpu_va)
124 return -ENOMEM;
125 }
126
127 if (flags & NVGPU_DMA_NO_KERNEL_MAPPING)
128 err = gk20a_get_sgtable_from_pages(d, &mem->sgt, mem->pages,
129 iova, size);
130 else {
131 err = gk20a_get_sgtable(d, &mem->sgt, mem->cpu_va, iova, size);
132 memset(mem->cpu_va, 0, size);
133 }
134 if (err)
135 goto fail_free;
136
137 mem->size = size;
138 mem->aperture = APERTURE_SYSMEM;
139 mem->flags = flags;
140
141 gk20a_dbg_fn("done");
142
143 return 0;
144
145fail_free:
146 dma_free_coherent(d, size, mem->cpu_va, iova);
147 mem->cpu_va = NULL;
148 mem->sgt = NULL;
149 return err;
150}
151
152int gk20a_gmmu_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem)
153{
154 return gk20a_gmmu_alloc_flags_vid(g,
155 NVGPU_DMA_NO_KERNEL_MAPPING, size, mem);
156}
157
158int gk20a_gmmu_alloc_flags_vid(struct gk20a *g, unsigned long flags,
159 size_t size, struct nvgpu_mem *mem)
160{
161 return gk20a_gmmu_alloc_flags_vid_at(g, flags, size, mem, 0);
162}
163
164int gk20a_gmmu_alloc_flags_vid_at(struct gk20a *g, unsigned long flags,
165 size_t size, struct nvgpu_mem *mem, dma_addr_t at)
166{
167#if defined(CONFIG_GK20A_VIDMEM)
168 u64 addr;
169 int err;
170 struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ?
171 &g->mm.vidmem.allocator :
172 &g->mm.vidmem.bootstrap_allocator;
173 int before_pending;
174
175 gk20a_dbg_fn("");
176
177 if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator))
178 return -ENOSYS;
179
180 /*
181 * Our own allocator doesn't have any flags yet, and we can't
182 * kernel-map these, so require explicit flags.
183 */
184 WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING);
185
186 nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
187 before_pending = atomic64_read(&g->mm.vidmem.bytes_pending);
188 addr = __gk20a_gmmu_alloc(vidmem_alloc, at, size);
189 nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
190 if (!addr) {
191 /*
192 * If memory is known to be freed soon, let the user know that
193 * it may be available after a while.
194 */
195 if (before_pending)
196 return -EAGAIN;
197 else
198 return -ENOMEM;
199 }
200
201 if (at)
202 mem->fixed = true;
203 else
204 mem->fixed = false;
205
206 mem->sgt = nvgpu_kzalloc(g, sizeof(struct sg_table));
207 if (!mem->sgt) {
208 err = -ENOMEM;
209 goto fail_physfree;
210 }
211
212 err = sg_alloc_table(mem->sgt, 1, GFP_KERNEL);
213 if (err)
214 goto fail_kfree;
215
216 set_vidmem_page_alloc(mem->sgt->sgl, addr);
217 sg_set_page(mem->sgt->sgl, NULL, size, 0);
218
219 mem->size = size;
220 mem->aperture = APERTURE_VIDMEM;
221 mem->allocator = vidmem_alloc;
222 mem->flags = flags;
223
224 nvgpu_init_list_node(&mem->clear_list_entry);
225
226 gk20a_dbg_fn("done at 0x%llx size %zu", addr, size);
227
228 return 0;
229
230fail_kfree:
231 nvgpu_kfree(g, mem->sgt);
232fail_physfree:
233 nvgpu_free(&g->mm.vidmem.allocator, addr);
234 return err;
235#else
236 return -ENOSYS;
237#endif
238}
239
240int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size,
241 struct nvgpu_mem *mem)
242{
243 return gk20a_gmmu_alloc_map_flags(vm, 0, size, mem);
244}
245
246int gk20a_gmmu_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags,
247 size_t size, struct nvgpu_mem *mem)
248{
249 if (vm->mm->vidmem_is_vidmem) {
250 /*
251 * Force the no-kernel-mapping flag on because we don't support
252 * the lack of it for vidmem - the user should not care when
253 * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a
254 * difference, the user should use the flag explicitly anyway.
255 */
256 int err = gk20a_gmmu_alloc_map_flags_vid(vm,
257 flags | NVGPU_DMA_NO_KERNEL_MAPPING,
258 size, mem);
259
260 if (!err)
261 return 0;
262 /*
263 * Fall back to sysmem (which may then also fail) in case
264 * vidmem is exhausted.
265 */
266 }
267
268 return gk20a_gmmu_alloc_map_flags_sys(vm, flags, size, mem);
269}
270
271int gk20a_gmmu_alloc_map_sys(struct vm_gk20a *vm, size_t size,
272 struct nvgpu_mem *mem)
273{
274 return gk20a_gmmu_alloc_map_flags_sys(vm, 0, size, mem);
275}
276
277int gk20a_gmmu_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags,
278 size_t size, struct nvgpu_mem *mem)
279{
280 int err = gk20a_gmmu_alloc_flags_sys(vm->mm->g, flags, size, mem);
281
282 if (err)
283 return err;
284
285 mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0,
286 gk20a_mem_flag_none, false,
287 mem->aperture);
288 if (!mem->gpu_va) {
289 err = -ENOMEM;
290 goto fail_free;
291 }
292
293 return 0;
294
295fail_free:
296 gk20a_gmmu_free(vm->mm->g, mem);
297 return err;
298}
299
300int gk20a_gmmu_alloc_map_vid(struct vm_gk20a *vm, size_t size,
301 struct nvgpu_mem *mem)
302{
303 return gk20a_gmmu_alloc_map_flags_vid(vm,
304 NVGPU_DMA_NO_KERNEL_MAPPING, size, mem);
305}
306
307int gk20a_gmmu_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags,
308 size_t size, struct nvgpu_mem *mem)
309{
310 int err = gk20a_gmmu_alloc_flags_vid(vm->mm->g, flags, size, mem);
311
312 if (err)
313 return err;
314
315 mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0,
316 gk20a_mem_flag_none, false,
317 mem->aperture);
318 if (!mem->gpu_va) {
319 err = -ENOMEM;
320 goto fail_free;
321 }
322
323 return 0;
324
325fail_free:
326 gk20a_gmmu_free(vm->mm->g, mem);
327 return err;
328}
329
330static void gk20a_gmmu_free_sys(struct gk20a *g, struct nvgpu_mem *mem)
331{
332 struct device *d = dev_from_gk20a(g);
333
334 if (mem->cpu_va || mem->pages) {
335 if (mem->flags) {
336 DEFINE_DMA_ATTRS(dma_attrs);
337
338 gk20a_dma_flags_to_attrs(&dma_attrs, mem->flags);
339
340 if (mem->flags & NVGPU_DMA_NO_KERNEL_MAPPING) {
341 dma_free_attrs(d, mem->size, mem->pages,
342 sg_dma_address(mem->sgt->sgl),
343 __DMA_ATTR(dma_attrs));
344 } else {
345 dma_free_attrs(d, mem->size, mem->cpu_va,
346 sg_dma_address(mem->sgt->sgl),
347 __DMA_ATTR(dma_attrs));
348 }
349 } else {
350 dma_free_coherent(d, mem->size, mem->cpu_va,
351 sg_dma_address(mem->sgt->sgl));
352 }
353 mem->cpu_va = NULL;
354 mem->pages = NULL;
355 }
356
357 if (mem->sgt)
358 gk20a_free_sgtable(g, &mem->sgt);
359
360 mem->size = 0;
361 mem->aperture = APERTURE_INVALID;
362}
363
364static void gk20a_gmmu_free_vid(struct gk20a *g, struct nvgpu_mem *mem)
365{
366#if defined(CONFIG_GK20A_VIDMEM)
367 bool was_empty;
368
369 /* Sanity check - only this supported when allocating. */
370 WARN_ON(mem->flags != NVGPU_DMA_NO_KERNEL_MAPPING);
371
372 if (mem->user_mem) {
373 nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
374 was_empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head);
375 nvgpu_list_add_tail(&mem->clear_list_entry,
376 &g->mm.vidmem.clear_list_head);
377 atomic64_add(mem->size, &g->mm.vidmem.bytes_pending);
378 nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
379
380 if (was_empty) {
381 cancel_work_sync(&g->mm.vidmem.clear_mem_worker);
382 schedule_work(&g->mm.vidmem.clear_mem_worker);
383 }
384 } else {
385 nvgpu_memset(g, mem, 0, 0, mem->size);
386 nvgpu_free(mem->allocator,
387 (u64)get_vidmem_page_alloc(mem->sgt->sgl));
388 gk20a_free_sgtable(g, &mem->sgt);
389
390 mem->size = 0;
391 mem->aperture = APERTURE_INVALID;
392 }
393#endif
394}
395
396void gk20a_gmmu_free(struct gk20a *g, struct nvgpu_mem *mem)
397{
398 switch (mem->aperture) {
399 case APERTURE_SYSMEM:
400 return gk20a_gmmu_free_sys(g, mem);
401 case APERTURE_VIDMEM:
402 return gk20a_gmmu_free_vid(g, mem);
403 default:
404 break; /* like free() on "null" memory */
405 }
406}
407
408void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem)
409{
410 if (mem->gpu_va)
411 gk20a_gmmu_unmap(vm, mem->gpu_va, mem->size, gk20a_mem_flag_none);
412 mem->gpu_va = 0;
413
414 gk20a_gmmu_free(vm->mm->g, mem);
415}
diff --git a/drivers/gpu/nvgpu/common/semaphore.c b/drivers/gpu/nvgpu/common/semaphore.c
index 6fb6c27e..cfe1149f 100644
--- a/drivers/gpu/nvgpu/common/semaphore.c
+++ b/drivers/gpu/nvgpu/common/semaphore.c
@@ -18,6 +18,7 @@
18#include <linux/dma-mapping.h> 18#include <linux/dma-mapping.h>
19#include <linux/highmem.h> 19#include <linux/highmem.h>
20 20
21#include <nvgpu/dma.h>
21#include <nvgpu/semaphore.h> 22#include <nvgpu/semaphore.h>
22#include <nvgpu/kmem.h> 23#include <nvgpu/kmem.h>
23 24
diff --git a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
index 0db6c21a..e70ee4a6 100644
--- a/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/cde_gk20a.c
@@ -24,6 +24,7 @@
24 24
25#include <trace/events/gk20a.h> 25#include <trace/events/gk20a.h>
26 26
27#include <nvgpu/dma.h>
27#include <nvgpu/timers.h> 28#include <nvgpu/timers.h>
28#include <nvgpu/nvgpu_common.h> 29#include <nvgpu/nvgpu_common.h>
29#include <nvgpu/kmem.h> 30#include <nvgpu/kmem.h>
diff --git a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
index ce76bfc3..9cc4b678 100644
--- a/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/ce2_gk20a.c
@@ -26,6 +26,7 @@
26#include <linux/debugfs.h> 26#include <linux/debugfs.h>
27 27
28#include <nvgpu/kmem.h> 28#include <nvgpu/kmem.h>
29#include <nvgpu/dma.h>
29 30
30#include "gk20a.h" 31#include "gk20a.h"
31#include "debug_gk20a.h" 32#include "debug_gk20a.h"
diff --git a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
index d0e2be79..6be616b3 100644
--- a/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/channel_gk20a.c
@@ -26,6 +26,7 @@
26#include <nvgpu/semaphore.h> 26#include <nvgpu/semaphore.h>
27#include <nvgpu/timers.h> 27#include <nvgpu/timers.h>
28#include <nvgpu/kmem.h> 28#include <nvgpu/kmem.h>
29#include <nvgpu/dma.h>
29 30
30#include "gk20a.h" 31#include "gk20a.h"
31#include "debug_gk20a.h" 32#include "debug_gk20a.h"
diff --git a/drivers/gpu/nvgpu/gk20a/css_gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/css_gr_gk20a.c
index 98fa53ab..738e8c1c 100644
--- a/drivers/gpu/nvgpu/gk20a/css_gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/css_gr_gk20a.c
@@ -22,6 +22,7 @@
22 22
23#include <nvgpu/kmem.h> 23#include <nvgpu/kmem.h>
24#include <nvgpu/lock.h> 24#include <nvgpu/lock.h>
25#include <nvgpu/dma.h>
25 26
26#include "gk20a.h" 27#include "gk20a.h"
27#include "css_gr_gk20a.h" 28#include "css_gr_gk20a.h"
diff --git a/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c b/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
index 3ed28718..d8fa7505 100644
--- a/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fecs_trace_gk20a.c
@@ -24,6 +24,7 @@
24#include <uapi/linux/nvgpu.h> 24#include <uapi/linux/nvgpu.h>
25 25
26#include <nvgpu/kmem.h> 26#include <nvgpu/kmem.h>
27#include <nvgpu/dma.h>
27 28
28#include "ctxsw_trace_gk20a.h" 29#include "ctxsw_trace_gk20a.h"
29#include "fecs_trace_gk20a.h" 30#include "fecs_trace_gk20a.h"
diff --git a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
index 1e9a8e15..c1f94eb3 100644
--- a/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/fifo_gk20a.c
@@ -26,6 +26,7 @@
26#endif 26#endif
27#include <linux/sort.h> 27#include <linux/sort.h>
28 28
29#include <nvgpu/dma.h>
29#include <nvgpu/timers.h> 30#include <nvgpu/timers.h>
30#include <nvgpu/semaphore.h> 31#include <nvgpu/semaphore.h>
31#include <nvgpu/kmem.h> 32#include <nvgpu/kmem.h>
diff --git a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
index 971e2320..a9b6a546 100644
--- a/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/gr_gk20a.c
@@ -29,6 +29,7 @@
29#include <linux/bsearch.h> 29#include <linux/bsearch.h>
30#include <trace/events/gk20a.h> 30#include <trace/events/gk20a.h>
31 31
32#include <nvgpu/dma.h>
32#include <nvgpu/kmem.h> 33#include <nvgpu/kmem.h>
33#include <nvgpu/timers.h> 34#include <nvgpu/timers.h>
34#include <nvgpu/nvgpu_common.h> 35#include <nvgpu/nvgpu_common.h>
diff --git a/drivers/gpu/nvgpu/gk20a/ltc_common.c b/drivers/gpu/nvgpu/gk20a/ltc_common.c
index 7c4db84e..7c73be77 100644
--- a/drivers/gpu/nvgpu/gk20a/ltc_common.c
+++ b/drivers/gpu/nvgpu/gk20a/ltc_common.c
@@ -21,6 +21,8 @@
21#include <linux/dma-mapping.h> 21#include <linux/dma-mapping.h>
22#include <linux/delay.h> 22#include <linux/delay.h>
23 23
24#include <nvgpu/dma.h>
25
24#include "gk20a.h" 26#include "gk20a.h"
25#include "gr_gk20a.h" 27#include "gr_gk20a.h"
26 28
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
index cdd0e541..79654af3 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.c
@@ -30,6 +30,7 @@
30#include <uapi/linux/nvgpu.h> 30#include <uapi/linux/nvgpu.h>
31#include <trace/events/gk20a.h> 31#include <trace/events/gk20a.h>
32 32
33#include <nvgpu/dma.h>
33#include <nvgpu/kmem.h> 34#include <nvgpu/kmem.h>
34#include <nvgpu/timers.h> 35#include <nvgpu/timers.h>
35#include <nvgpu/pramin.h> 36#include <nvgpu/pramin.h>
@@ -2519,152 +2520,6 @@ u64 gk20a_gmmu_fixed_map(struct vm_gk20a *vm,
2519 aperture); 2520 aperture);
2520} 2521}
2521 2522
2522int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem)
2523{
2524 return gk20a_gmmu_alloc_flags(g, 0, size, mem);
2525}
2526
2527int gk20a_gmmu_alloc_flags(struct gk20a *g, unsigned long flags, size_t size,
2528 struct nvgpu_mem *mem)
2529{
2530 if (g->mm.vidmem_is_vidmem) {
2531 /*
2532 * Force the no-kernel-mapping flag on because we don't support
2533 * the lack of it for vidmem - the user should not care when
2534 * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a
2535 * difference, the user should use the flag explicitly anyway.
2536 */
2537 int err = gk20a_gmmu_alloc_flags_vid(g,
2538 flags | NVGPU_DMA_NO_KERNEL_MAPPING,
2539 size, mem);
2540
2541 if (!err)
2542 return 0;
2543 /*
2544 * Fall back to sysmem (which may then also fail) in case
2545 * vidmem is exhausted.
2546 */
2547 }
2548
2549 return gk20a_gmmu_alloc_flags_sys(g, flags, size, mem);
2550}
2551
2552int gk20a_gmmu_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem)
2553{
2554 return gk20a_gmmu_alloc_flags_sys(g, 0, size, mem);
2555}
2556
2557#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)
2558static void gk20a_dma_flags_to_attrs(unsigned long *attrs,
2559 unsigned long flags)
2560#define ATTR_ARG(x) *x
2561#else
2562static void gk20a_dma_flags_to_attrs(struct dma_attrs *attrs,
2563 unsigned long flags)
2564#define ATTR_ARG(x) x
2565#endif
2566{
2567 if (flags & NVGPU_DMA_NO_KERNEL_MAPPING)
2568 dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs));
2569 if (flags & NVGPU_DMA_FORCE_CONTIGUOUS)
2570 dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs));
2571 if (flags & NVGPU_DMA_READ_ONLY)
2572 dma_set_attr(DMA_ATTR_READ_ONLY, ATTR_ARG(attrs));
2573#undef ATTR_ARG
2574}
2575
2576int gk20a_gmmu_alloc_flags_sys(struct gk20a *g, unsigned long flags,
2577 size_t size, struct nvgpu_mem *mem)
2578{
2579 struct device *d = dev_from_gk20a(g);
2580 int err;
2581 dma_addr_t iova;
2582
2583 gk20a_dbg_fn("");
2584
2585 if (flags) {
2586 DEFINE_DMA_ATTRS(dma_attrs);
2587
2588 gk20a_dma_flags_to_attrs(&dma_attrs, flags);
2589
2590 if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) {
2591 mem->pages = dma_alloc_attrs(d,
2592 size, &iova, GFP_KERNEL,
2593 __DMA_ATTR(dma_attrs));
2594 if (!mem->pages)
2595 return -ENOMEM;
2596 } else {
2597 mem->cpu_va = dma_alloc_attrs(d,
2598 size, &iova, GFP_KERNEL,
2599 __DMA_ATTR(dma_attrs));
2600 if (!mem->cpu_va)
2601 return -ENOMEM;
2602 }
2603 } else {
2604 mem->cpu_va = dma_alloc_coherent(d, size, &iova, GFP_KERNEL);
2605 if (!mem->cpu_va)
2606 return -ENOMEM;
2607 }
2608
2609 if (flags & NVGPU_DMA_NO_KERNEL_MAPPING)
2610 err = gk20a_get_sgtable_from_pages(d, &mem->sgt, mem->pages,
2611 iova, size);
2612 else {
2613 err = gk20a_get_sgtable(d, &mem->sgt, mem->cpu_va, iova, size);
2614 memset(mem->cpu_va, 0, size);
2615 }
2616 if (err)
2617 goto fail_free;
2618
2619 mem->size = size;
2620 mem->aperture = APERTURE_SYSMEM;
2621 mem->flags = flags;
2622
2623 gk20a_dbg_fn("done");
2624
2625 return 0;
2626
2627fail_free:
2628 dma_free_coherent(d, size, mem->cpu_va, iova);
2629 mem->cpu_va = NULL;
2630 mem->sgt = NULL;
2631 return err;
2632}
2633
2634static void gk20a_gmmu_free_sys(struct gk20a *g, struct nvgpu_mem *mem)
2635{
2636 struct device *d = dev_from_gk20a(g);
2637
2638 if (mem->cpu_va || mem->pages) {
2639 if (mem->flags) {
2640 DEFINE_DMA_ATTRS(dma_attrs);
2641
2642 gk20a_dma_flags_to_attrs(&dma_attrs, mem->flags);
2643
2644 if (mem->flags & NVGPU_DMA_NO_KERNEL_MAPPING) {
2645 dma_free_attrs(d, mem->size, mem->pages,
2646 sg_dma_address(mem->sgt->sgl),
2647 __DMA_ATTR(dma_attrs));
2648 } else {
2649 dma_free_attrs(d, mem->size, mem->cpu_va,
2650 sg_dma_address(mem->sgt->sgl),
2651 __DMA_ATTR(dma_attrs));
2652 }
2653 } else {
2654 dma_free_coherent(d, mem->size, mem->cpu_va,
2655 sg_dma_address(mem->sgt->sgl));
2656 }
2657 mem->cpu_va = NULL;
2658 mem->pages = NULL;
2659 }
2660
2661 if (mem->sgt)
2662 gk20a_free_sgtable(g, &mem->sgt);
2663
2664 mem->size = 0;
2665 mem->aperture = APERTURE_INVALID;
2666}
2667
2668#if defined(CONFIG_GK20A_VIDMEM) 2523#if defined(CONFIG_GK20A_VIDMEM)
2669static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem) 2524static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem)
2670{ 2525{
@@ -2728,153 +2583,6 @@ static int gk20a_gmmu_clear_vidmem_mem(struct gk20a *g, struct nvgpu_mem *mem)
2728} 2583}
2729#endif 2584#endif
2730 2585
2731int gk20a_gmmu_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem)
2732{
2733 return gk20a_gmmu_alloc_flags_vid(g,
2734 NVGPU_DMA_NO_KERNEL_MAPPING, size, mem);
2735}
2736
2737int gk20a_gmmu_alloc_flags_vid(struct gk20a *g, unsigned long flags,
2738 size_t size, struct nvgpu_mem *mem)
2739{
2740 return gk20a_gmmu_alloc_flags_vid_at(g, flags, size, mem, 0);
2741}
2742
2743#if defined(CONFIG_GK20A_VIDMEM)
2744static u64 __gk20a_gmmu_alloc(struct nvgpu_allocator *allocator, dma_addr_t at,
2745 size_t size)
2746{
2747 u64 addr = 0;
2748
2749 if (at)
2750 addr = nvgpu_alloc_fixed(allocator, at, size, 0);
2751 else
2752 addr = nvgpu_alloc(allocator, size);
2753
2754 return addr;
2755}
2756#endif
2757
2758int gk20a_gmmu_alloc_flags_vid_at(struct gk20a *g, unsigned long flags,
2759 size_t size, struct nvgpu_mem *mem, dma_addr_t at)
2760{
2761#if defined(CONFIG_GK20A_VIDMEM)
2762 u64 addr;
2763 int err;
2764 struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ?
2765 &g->mm.vidmem.allocator :
2766 &g->mm.vidmem.bootstrap_allocator;
2767 int before_pending;
2768
2769 gk20a_dbg_fn("");
2770
2771 if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator))
2772 return -ENOSYS;
2773
2774 /*
2775 * Our own allocator doesn't have any flags yet, and we can't
2776 * kernel-map these, so require explicit flags.
2777 */
2778 WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING);
2779
2780 nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
2781 before_pending = atomic64_read(&g->mm.vidmem.bytes_pending);
2782 addr = __gk20a_gmmu_alloc(vidmem_alloc, at, size);
2783 nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
2784 if (!addr) {
2785 /*
2786 * If memory is known to be freed soon, let the user know that
2787 * it may be available after a while.
2788 */
2789 if (before_pending)
2790 return -EAGAIN;
2791 else
2792 return -ENOMEM;
2793 }
2794
2795 if (at)
2796 mem->fixed = true;
2797 else
2798 mem->fixed = false;
2799
2800 mem->sgt = nvgpu_kzalloc(g, sizeof(struct sg_table));
2801 if (!mem->sgt) {
2802 err = -ENOMEM;
2803 goto fail_physfree;
2804 }
2805
2806 err = sg_alloc_table(mem->sgt, 1, GFP_KERNEL);
2807 if (err)
2808 goto fail_kfree;
2809
2810 set_vidmem_page_alloc(mem->sgt->sgl, addr);
2811 sg_set_page(mem->sgt->sgl, NULL, size, 0);
2812
2813 mem->size = size;
2814 mem->aperture = APERTURE_VIDMEM;
2815 mem->allocator = vidmem_alloc;
2816 mem->flags = flags;
2817
2818 nvgpu_init_list_node(&mem->clear_list_entry);
2819
2820 gk20a_dbg_fn("done at 0x%llx size %zu", addr, size);
2821
2822 return 0;
2823
2824fail_kfree:
2825 nvgpu_kfree(g, mem->sgt);
2826fail_physfree:
2827 nvgpu_free(&g->mm.vidmem.allocator, addr);
2828 return err;
2829#else
2830 return -ENOSYS;
2831#endif
2832}
2833
2834static void gk20a_gmmu_free_vid(struct gk20a *g, struct nvgpu_mem *mem)
2835{
2836#if defined(CONFIG_GK20A_VIDMEM)
2837 bool was_empty;
2838
2839 /* Sanity check - only this supported when allocating. */
2840 WARN_ON(mem->flags != NVGPU_DMA_NO_KERNEL_MAPPING);
2841
2842 if (mem->user_mem) {
2843 nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex);
2844 was_empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head);
2845 nvgpu_list_add_tail(&mem->clear_list_entry,
2846 &g->mm.vidmem.clear_list_head);
2847 atomic64_add(mem->size, &g->mm.vidmem.bytes_pending);
2848 nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex);
2849
2850 if (was_empty) {
2851 cancel_work_sync(&g->mm.vidmem.clear_mem_worker);
2852 schedule_work(&g->mm.vidmem.clear_mem_worker);
2853 }
2854 } else {
2855 nvgpu_memset(g, mem, 0, 0, mem->size);
2856 nvgpu_free(mem->allocator,
2857 (u64)get_vidmem_page_alloc(mem->sgt->sgl));
2858 gk20a_free_sgtable(g, &mem->sgt);
2859
2860 mem->size = 0;
2861 mem->aperture = APERTURE_INVALID;
2862 }
2863#endif
2864}
2865
2866void gk20a_gmmu_free(struct gk20a *g, struct nvgpu_mem *mem)
2867{
2868 switch (mem->aperture) {
2869 case APERTURE_SYSMEM:
2870 return gk20a_gmmu_free_sys(g, mem);
2871 case APERTURE_VIDMEM:
2872 return gk20a_gmmu_free_vid(g, mem);
2873 default:
2874 break; /* like free() on "null" memory */
2875 }
2876}
2877
2878/* 2586/*
2879 * If mem is in VIDMEM, return base address in vidmem 2587 * If mem is in VIDMEM, return base address in vidmem
2880 * else return IOVA address for SYSMEM 2588 * else return IOVA address for SYSMEM
@@ -2938,105 +2646,6 @@ static void gk20a_vidmem_clear_mem_worker(struct work_struct *work)
2938} 2646}
2939#endif 2647#endif
2940 2648
2941int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size,
2942 struct nvgpu_mem *mem)
2943{
2944 return gk20a_gmmu_alloc_map_flags(vm, 0, size, mem);
2945}
2946
2947int gk20a_gmmu_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags,
2948 size_t size, struct nvgpu_mem *mem)
2949{
2950 if (vm->mm->vidmem_is_vidmem) {
2951 /*
2952 * Force the no-kernel-mapping flag on because we don't support
2953 * the lack of it for vidmem - the user should not care when
2954 * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a
2955 * difference, the user should use the flag explicitly anyway.
2956 */
2957 int err = gk20a_gmmu_alloc_map_flags_vid(vm,
2958 flags | NVGPU_DMA_NO_KERNEL_MAPPING,
2959 size, mem);
2960
2961 if (!err)
2962 return 0;
2963 /*
2964 * Fall back to sysmem (which may then also fail) in case
2965 * vidmem is exhausted.
2966 */
2967 }
2968
2969 return gk20a_gmmu_alloc_map_flags_sys(vm, flags, size, mem);
2970}
2971
2972int gk20a_gmmu_alloc_map_sys(struct vm_gk20a *vm, size_t size,
2973 struct nvgpu_mem *mem)
2974{
2975 return gk20a_gmmu_alloc_map_flags_sys(vm, 0, size, mem);
2976}
2977
2978int gk20a_gmmu_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags,
2979 size_t size, struct nvgpu_mem *mem)
2980{
2981 int err = gk20a_gmmu_alloc_flags_sys(vm->mm->g, flags, size, mem);
2982
2983 if (err)
2984 return err;
2985
2986 mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0,
2987 gk20a_mem_flag_none, false,
2988 mem->aperture);
2989 if (!mem->gpu_va) {
2990 err = -ENOMEM;
2991 goto fail_free;
2992 }
2993
2994 return 0;
2995
2996fail_free:
2997 gk20a_gmmu_free(vm->mm->g, mem);
2998 return err;
2999}
3000
3001int gk20a_gmmu_alloc_map_vid(struct vm_gk20a *vm, size_t size,
3002 struct nvgpu_mem *mem)
3003{
3004 return gk20a_gmmu_alloc_map_flags_vid(vm,
3005 NVGPU_DMA_NO_KERNEL_MAPPING, size, mem);
3006}
3007
3008int gk20a_gmmu_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags,
3009 size_t size, struct nvgpu_mem *mem)
3010{
3011 int err = gk20a_gmmu_alloc_flags_vid(vm->mm->g, flags, size, mem);
3012
3013 if (err)
3014 return err;
3015
3016 mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0,
3017 gk20a_mem_flag_none, false,
3018 mem->aperture);
3019 if (!mem->gpu_va) {
3020 err = -ENOMEM;
3021 goto fail_free;
3022 }
3023
3024 return 0;
3025
3026fail_free:
3027 gk20a_gmmu_free(vm->mm->g, mem);
3028 return err;
3029}
3030
3031void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem)
3032{
3033 if (mem->gpu_va)
3034 gk20a_gmmu_unmap(vm, mem->gpu_va, mem->size, gk20a_mem_flag_none);
3035 mem->gpu_va = 0;
3036
3037 gk20a_gmmu_free(vm->mm->g, mem);
3038}
3039
3040dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr) 2649dma_addr_t gk20a_mm_gpuva_to_iova_base(struct vm_gk20a *vm, u64 gpu_vaddr)
3041{ 2650{
3042 struct mapped_buffer_node *buffer; 2651 struct mapped_buffer_node *buffer;
diff --git a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
index db72ca79..53366caf 100644
--- a/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/mm_gk20a.h
@@ -524,56 +524,6 @@ u64 gk20a_gmmu_fixed_map(struct vm_gk20a *vm,
524 bool priv, 524 bool priv,
525 enum nvgpu_aperture aperture); 525 enum nvgpu_aperture aperture);
526 526
527/* Flags for the below gk20a_gmmu_{alloc,alloc_map}_flags* */
528
529/*
530 * Don't create a virtual kernel mapping for the buffer but only allocate it;
531 * this may save some resources. The buffer can be mapped later explicitly.
532 */
533#define NVGPU_DMA_NO_KERNEL_MAPPING (1 << 0)
534/*
535 * Don't allow building the buffer from individual pages but require a
536 * physically contiguous block.
537 */
538#define NVGPU_DMA_FORCE_CONTIGUOUS (1 << 1)
539/*
540 * Make the mapping read-only.
541 */
542#define NVGPU_DMA_READ_ONLY (1 << 2)
543
544int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size,
545 struct nvgpu_mem *mem);
546int gk20a_gmmu_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags,
547 size_t size, struct nvgpu_mem *mem);
548
549int gk20a_gmmu_alloc_map_sys(struct vm_gk20a *vm, size_t size,
550 struct nvgpu_mem *mem);
551int gk20a_gmmu_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags,
552 size_t size, struct nvgpu_mem *mem);
553
554int gk20a_gmmu_alloc_map_vid(struct vm_gk20a *vm, size_t size,
555 struct nvgpu_mem *mem);
556int gk20a_gmmu_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags,
557 size_t size, struct nvgpu_mem *mem);
558
559void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem);
560
561int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem);
562int gk20a_gmmu_alloc_flags(struct gk20a *g, unsigned long flags, size_t size,
563 struct nvgpu_mem *mem);
564
565int gk20a_gmmu_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem);
566int gk20a_gmmu_alloc_flags_sys(struct gk20a *g, unsigned long flags,
567 size_t size, struct nvgpu_mem *mem);
568
569int gk20a_gmmu_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem);
570int gk20a_gmmu_alloc_flags_vid(struct gk20a *g, unsigned long flags,
571 size_t size, struct nvgpu_mem *mem);
572int gk20a_gmmu_alloc_flags_vid_at(struct gk20a *g, unsigned long flags,
573 size_t size, struct nvgpu_mem *mem, dma_addr_t at);
574
575void gk20a_gmmu_free(struct gk20a *g, struct nvgpu_mem *mem);
576
577static inline phys_addr_t gk20a_mem_phys(struct nvgpu_mem *mem) 527static inline phys_addr_t gk20a_mem_phys(struct nvgpu_mem *mem)
578{ 528{
579 /* FIXME: the sgt/sgl may get null if this is accessed e.g. in an isr 529 /* FIXME: the sgt/sgl may get null if this is accessed e.g. in an isr
diff --git a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
index 591b7163..7a6bfe22 100644
--- a/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
+++ b/drivers/gpu/nvgpu/gk20a/pmu_gk20a.c
@@ -26,6 +26,7 @@
26#include <nvgpu/nvgpu_common.h> 26#include <nvgpu/nvgpu_common.h>
27#include <nvgpu/timers.h> 27#include <nvgpu/timers.h>
28#include <nvgpu/kmem.h> 28#include <nvgpu/kmem.h>
29#include <nvgpu/dma.h>
29 30
30#include "gk20a.h" 31#include "gk20a.h"
31#include "gr_gk20a.h" 32#include "gr_gk20a.h"
diff --git a/drivers/gpu/nvgpu/gm20b/acr_gm20b.c b/drivers/gpu/nvgpu/gm20b/acr_gm20b.c
index 013ce43a..3cfcbb19 100644
--- a/drivers/gpu/nvgpu/gm20b/acr_gm20b.c
+++ b/drivers/gpu/nvgpu/gm20b/acr_gm20b.c
@@ -20,6 +20,7 @@
20 20
21#include <linux/platform/tegra/mc.h> 21#include <linux/platform/tegra/mc.h>
22 22
23#include <nvgpu/dma.h>
23#include <nvgpu/timers.h> 24#include <nvgpu/timers.h>
24#include <nvgpu/nvgpu_common.h> 25#include <nvgpu/nvgpu_common.h>
25#include <nvgpu/kmem.h> 26#include <nvgpu/kmem.h>
diff --git a/drivers/gpu/nvgpu/gp106/acr_gp106.c b/drivers/gpu/nvgpu/gp106/acr_gp106.c
index 41c4981d..9acc8eda 100644
--- a/drivers/gpu/nvgpu/gp106/acr_gp106.c
+++ b/drivers/gpu/nvgpu/gp106/acr_gp106.c
@@ -20,6 +20,7 @@
20 20
21#include <nvgpu/nvgpu_common.h> 21#include <nvgpu/nvgpu_common.h>
22#include <nvgpu/kmem.h> 22#include <nvgpu/kmem.h>
23#include <nvgpu/dma.h>
23#include <nvgpu/acr/nvgpu_acr.h> 24#include <nvgpu/acr/nvgpu_acr.h>
24 25
25#include "gk20a/gk20a.h" 26#include "gk20a/gk20a.h"
diff --git a/drivers/gpu/nvgpu/gp106/gr_gp106.c b/drivers/gpu/nvgpu/gp106/gr_gp106.c
index dae23374..78859f88 100644
--- a/drivers/gpu/nvgpu/gp106/gr_gp106.c
+++ b/drivers/gpu/nvgpu/gp106/gr_gp106.c
@@ -13,6 +13,8 @@
13 * more details. 13 * more details.
14 */ 14 */
15 15
16#include <nvgpu/dma.h>
17
16#include "gk20a/gk20a.h" 18#include "gk20a/gk20a.h"
17#include "gk20a/gr_gk20a.h" 19#include "gk20a/gr_gk20a.h"
18#include "gm20b/gr_gm20b.h" 20#include "gm20b/gr_gm20b.h"
diff --git a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
index a7e77232..b305b895 100644
--- a/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/fifo_gp10b.c
@@ -15,6 +15,8 @@
15 15
16#include <linux/delay.h> 16#include <linux/delay.h>
17 17
18#include <nvgpu/dma.h>
19
18#include "fifo_gp10b.h" 20#include "fifo_gp10b.h"
19 21
20#include "gk20a/gk20a.h" 22#include "gk20a/gk20a.h"
diff --git a/drivers/gpu/nvgpu/gp10b/gr_gp10b.c b/drivers/gpu/nvgpu/gp10b/gr_gp10b.c
index 8e1517f6..b9367120 100644
--- a/drivers/gpu/nvgpu/gp10b/gr_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/gr_gp10b.c
@@ -22,6 +22,7 @@
22 22
23#include <nvgpu/timers.h> 23#include <nvgpu/timers.h>
24#include <nvgpu/kmem.h> 24#include <nvgpu/kmem.h>
25#include <nvgpu/dma.h>
25 26
26#include "gk20a/gk20a.h" 27#include "gk20a/gk20a.h"
27#include "gk20a/gr_gk20a.h" 28#include "gk20a/gr_gk20a.h"
diff --git a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
index a0dc8c55..2f894435 100644
--- a/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/mm_gp10b.c
@@ -16,6 +16,8 @@
16#include <linux/pm_runtime.h> 16#include <linux/pm_runtime.h>
17#include <linux/dma-mapping.h> 17#include <linux/dma-mapping.h>
18 18
19#include <nvgpu/dma.h>
20
19#include "gk20a/gk20a.h" 21#include "gk20a/gk20a.h"
20#include "gm20b/mm_gm20b.h" 22#include "gm20b/mm_gm20b.h"
21#include "mm_gp10b.h" 23#include "mm_gp10b.h"
diff --git a/drivers/gpu/nvgpu/gp10b/rpfb_gp10b.c b/drivers/gpu/nvgpu/gp10b/rpfb_gp10b.c
index e73bcd8f..bf52b5c9 100644
--- a/drivers/gpu/nvgpu/gp10b/rpfb_gp10b.c
+++ b/drivers/gpu/nvgpu/gp10b/rpfb_gp10b.c
@@ -16,6 +16,8 @@
16#include <linux/pm_runtime.h> 16#include <linux/pm_runtime.h>
17#include <linux/dma-mapping.h> 17#include <linux/dma-mapping.h>
18 18
19#include <nvgpu/dma.h>
20
19#include "gk20a/gk20a.h" 21#include "gk20a/gk20a.h"
20 22
21#include "rpfb_gp10b.h" 23#include "rpfb_gp10b.h"
diff --git a/drivers/gpu/nvgpu/include/nvgpu/dma.h b/drivers/gpu/nvgpu/include/nvgpu/dma.h
new file mode 100644
index 00000000..d4fad584
--- /dev/null
+++ b/drivers/gpu/nvgpu/include/nvgpu/dma.h
@@ -0,0 +1,312 @@
1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#ifndef __NVGPU_DMA_H__
18#define __NVGPU_DMA_H__
19
20#include <nvgpu/types.h>
21
22struct gk20a;
23struct vm_gk20a;
24struct nvgpu_mem;
25
26/*
27 * Flags for the below gk20a_gmmu_{alloc,alloc_map}_flags*
28 */
29
30/*
31 * Don't create a virtual kernel mapping for the buffer but only allocate it;
32 * this may save some resources. The buffer can be mapped later explicitly.
33 */
34#define NVGPU_DMA_NO_KERNEL_MAPPING (1 << 0)
35
36/*
37 * Don't allow building the buffer from individual pages but require a
38 * physically contiguous block.
39 */
40#define NVGPU_DMA_FORCE_CONTIGUOUS (1 << 1)
41
42/*
43 * Make the mapping read-only.
44 */
45#define NVGPU_DMA_READ_ONLY (1 << 2)
46
47/**
48 * gk20a_gmmu_alloc - Allocate DMA memory
49 *
50 * @g - The GPU.
51 * @size - Size of the allocation in bytes.
52 * @mem - Struct for storing the allocation information.
53 *
54 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
55 * Returns 0 on success and a suitable error code when there's an error. This
56 * memory can be either placed in VIDMEM or SYSMEM, which ever is more
57 * convenient for the driver.
58 */
59int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem);
60
61/**
62 * gk20a_gmmu_alloc_flags - Allocate DMA memory
63 *
64 * @g - The GPU.
65 * @flags - Flags modifying the operation of the DMA allocation.
66 * @size - Size of the allocation in bytes.
67 * @mem - Struct for storing the allocation information.
68 *
69 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
70 * Returns 0 on success and a suitable error code when there's an error. This
71 * memory can be either placed in VIDMEM or SYSMEM, which ever is more
72 * convenient for the driver.
73 *
74 * The following flags are accepted:
75 *
76 * %NVGPU_DMA_NO_KERNEL_MAPPING
77 * %NVGPU_DMA_FORCE_CONTIGUOUS
78 * %NVGPU_DMA_READ_ONLY
79 */
80int gk20a_gmmu_alloc_flags(struct gk20a *g, unsigned long flags, size_t size,
81 struct nvgpu_mem *mem);
82
83/**
84 * gk20a_gmmu_alloc_sys - Allocate DMA memory
85 *
86 * @g - The GPU.
87 * @size - Size of the allocation in bytes.
88 * @mem - Struct for storing the allocation information.
89 *
90 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
91 * Returns 0 on success and a suitable error code when there's an error. This
92 * allocates memory specifically in SYSMEM.
93 */
94int gk20a_gmmu_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem);
95
96/**
97 * gk20a_gmmu_alloc_flags_sys - Allocate DMA memory
98 *
99 * @g - The GPU.
100 * @flags - Flags modifying the operation of the DMA allocation.
101 * @size - Size of the allocation in bytes.
102 * @mem - Struct for storing the allocation information.
103 *
104 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
105 * Returns 0 on success and a suitable error code when there's an error. This
106 * allocates memory specifically in SYSMEM.
107 *
108 * The following flags are accepted:
109 *
110 * %NVGPU_DMA_NO_KERNEL_MAPPING
111 * %NVGPU_DMA_FORCE_CONTIGUOUS
112 * %NVGPU_DMA_READ_ONLY
113 */
114int gk20a_gmmu_alloc_flags_sys(struct gk20a *g, unsigned long flags,
115 size_t size, struct nvgpu_mem *mem);
116
117/**
118 * gk20a_gmmu_alloc_vid - Allocate DMA memory
119 *
120 * @g - The GPU.
121 * @size - Size of the allocation in bytes.
122 * @mem - Struct for storing the allocation information.
123 *
124 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
125 * Returns 0 on success and a suitable error code when there's an error. This
126 * allocates memory specifically in VIDMEM.
127 */
128int gk20a_gmmu_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem);
129
130/**
131 * gk20a_gmmu_alloc_flags_vid - Allocate DMA memory
132 *
133 * @g - The GPU.
134 * @flags - Flags modifying the operation of the DMA allocation.
135 * @size - Size of the allocation in bytes.
136 * @mem - Struct for storing the allocation information.
137 *
138 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
139 * Returns 0 on success and a suitable error code when there's an error. This
140 * allocates memory specifically in VIDMEM.
141 *
142 * Only the following flags are accepted:
143 *
144 * %NVGPU_DMA_NO_KERNEL_MAPPING
145 *
146 */
147int gk20a_gmmu_alloc_flags_vid(struct gk20a *g, unsigned long flags,
148 size_t size, struct nvgpu_mem *mem);
149
150/**
151 * gk20a_gmmu_alloc_flags_vid_at - Allocate DMA memory
152 *
153 * @g - The GPU.
154 * @flags - Flags modifying the operation of the DMA allocation.
155 * @size - Size of the allocation in bytes.
156 * @mem - Struct for storing the allocation information.
157 * @at - A specific location to attempt to allocate memory from or 0 if the
158 * caller does not care what the address is.
159 *
160 * Allocate memory suitable for doing DMA. Store the allocation info in @mem.
161 * Returns 0 on success and a suitable error code when there's an error. This
162 * allocates memory specifically in VIDMEM.
163 *
164 * Only the following flags are accepted:
165 *
166 * %NVGPU_DMA_NO_KERNEL_MAPPING
167 */
168int gk20a_gmmu_alloc_flags_vid_at(struct gk20a *g, unsigned long flags,
169 size_t size, struct nvgpu_mem *mem, dma_addr_t at);
170
171/**
172 * gk20a_gmmu_free - Free a DMA allocation
173 *
174 * @g - The GPU.
175 * @mem - An allocation to free.
176 *
177 * Free memory created with any of:
178 *
179 * gk20a_gmmu_alloc()
180 * gk20a_gmmu_alloc_flags()
181 * gk20a_gmmu_alloc_sys()
182 * gk20a_gmmu_alloc_flags_sys()
183 * gk20a_gmmu_alloc_vid()
184 * gk20a_gmmu_alloc_flags_vid()
185 * gk20a_gmmu_alloc_flags_vid_at()
186 */
187void gk20a_gmmu_free(struct gk20a *g, struct nvgpu_mem *mem);
188
189/**
190 * gk20a_gmmu_alloc_map - Allocate DMA memory and map into GMMU.
191 *
192 * @vm - VM context for GMMU mapping.
193 * @size - Size of the allocation in bytes.
194 * @mem - Struct for storing the allocation information.
195 *
196 * Allocate memory suitable for doing DMA and map that memory into the GMMU.
197 * Note this is different than mapping it into the CPU. This memory can be
198 * either placed in VIDMEM or SYSMEM, which ever is more convenient for the
199 * driver.
200 */
201int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size,
202 struct nvgpu_mem *mem);
203
204/**
205 * gk20a_gmmu_alloc_map_flags - Allocate DMA memory and map into GMMU.
206 *
207 * @vm - VM context for GMMU mapping.
208 * @flags - Flags modifying the operation of the DMA allocation.
209 * @size - Size of the allocation in bytes.
210 * @mem - Struct for storing the allocation information.
211 *
212 * Allocate memory suitable for doing DMA and map that memory into the GMMU.
213 * Note this is different than mapping it into the CPU. This memory can be
214 * either placed in VIDMEM or SYSMEM, which ever is more convenient for the
215 * driver.
216 *
217 * This version passes @flags on to the underlying DMA allocation. The accepted
218 * flags are:
219 *
220 * %NVGPU_DMA_NO_KERNEL_MAPPING
221 * %NVGPU_DMA_FORCE_CONTIGUOUS
222 * %NVGPU_DMA_READ_ONLY
223 */
224int gk20a_gmmu_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags,
225 size_t size, struct nvgpu_mem *mem);
226
227/**
228 * gk20a_gmmu_alloc_map_sys - Allocate DMA memory and map into GMMU.
229 *
230 * @vm - VM context for GMMU mapping.
231 * @size - Size of the allocation in bytes.
232 * @mem - Struct for storing the allocation information.
233 *
234 * Allocate memory suitable for doing DMA and map that memory into the GMMU.
235 * This memory will be placed in SYSMEM.
236 */
237int gk20a_gmmu_alloc_map_sys(struct vm_gk20a *vm, size_t size,
238 struct nvgpu_mem *mem);
239
240/**
241 * gk20a_gmmu_alloc_map_flags_sys - Allocate DMA memory and map into GMMU.
242 *
243 * @vm - VM context for GMMU mapping.
244 * @flags - Flags modifying the operation of the DMA allocation.
245 * @size - Size of the allocation in bytes.
246 * @mem - Struct for storing the allocation information.
247 *
248 * Allocate memory suitable for doing DMA and map that memory into the GMMU.
249 * This memory will be placed in SYSMEM.
250 *
251 * This version passes @flags on to the underlying DMA allocation. The accepted
252 * flags are:
253 *
254 * %NVGPU_DMA_NO_KERNEL_MAPPING
255 * %NVGPU_DMA_FORCE_CONTIGUOUS
256 * %NVGPU_DMA_READ_ONLY
257 */
258int gk20a_gmmu_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags,
259 size_t size, struct nvgpu_mem *mem);
260
261/**
262 * gk20a_gmmu_alloc_map_vid - Allocate DMA memory and map into GMMU.
263 *
264 * @vm - VM context for GMMU mapping.
265 * @size - Size of the allocation in bytes.
266 * @mem - Struct for storing the allocation information.
267 *
268 * Allocate memory suitable for doing DMA and map that memory into the GMMU.
269 * This memory will be placed in VIDMEM.
270 */
271int gk20a_gmmu_alloc_map_vid(struct vm_gk20a *vm, size_t size,
272 struct nvgpu_mem *mem);
273
274/**
275 * gk20a_gmmu_alloc_map_flags_vid - Allocate DMA memory and map into GMMU.
276 *
277 * @vm - VM context for GMMU mapping.
278 * @flags - Flags modifying the operation of the DMA allocation.
279 * @size - Size of the allocation in bytes.
280 * @mem - Struct for storing the allocation information.
281 *
282 * Allocate memory suitable for doing DMA and map that memory into the GMMU.
283 * This memory will be placed in VIDMEM.
284 *
285 * This version passes @flags on to the underlying DMA allocation. The accepted
286 * flags are:
287 *
288 * %NVGPU_DMA_NO_KERNEL_MAPPING
289 * %NVGPU_DMA_FORCE_CONTIGUOUS
290 * %NVGPU_DMA_READ_ONLY
291 */
292int gk20a_gmmu_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags,
293 size_t size, struct nvgpu_mem *mem);
294
295/**
296 * gk20a_gmmu_unmap_free - Free a DMA allocation
297 *
298 * @g - The GPU.
299 * @mem - An allocation to free.
300 *
301 * Free memory created with any of:
302 *
303 * gk20a_gmmu_alloc_map()
304 * gk20a_gmmu_alloc_map_flags()
305 * gk20a_gmmu_alloc_map_sys()
306 * gk20a_gmmu_alloc_map_flags_sys()
307 * gk20a_gmmu_alloc_map_vid()
308 * gk20a_gmmu_alloc_map_flags_vid()
309 */
310void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem);
311
312#endif
diff --git a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
index cfe9322e..59fb0c4a 100644
--- a/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
+++ b/drivers/gpu/nvgpu/vgpu/fifo_vgpu.c
@@ -17,6 +17,7 @@
17#include <trace/events/gk20a.h> 17#include <trace/events/gk20a.h>
18 18
19#include <nvgpu/kmem.h> 19#include <nvgpu/kmem.h>
20#include <nvgpu/dma.h>
20 21
21#include "vgpu/vgpu.h" 22#include "vgpu/vgpu.h"
22#include "gk20a/ctxsw_trace_gk20a.h" 23#include "gk20a/ctxsw_trace_gk20a.h"
diff --git a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c
index 8cb5b029..527e12e4 100644
--- a/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c
+++ b/drivers/gpu/nvgpu/vgpu/gp10b/vgpu_gr_gp10b.c
@@ -12,6 +12,7 @@
12 */ 12 */
13 13
14#include <nvgpu/kmem.h> 14#include <nvgpu/kmem.h>
15#include <nvgpu/dma.h>
15 16
16#include "vgpu/vgpu.h" 17#include "vgpu/vgpu.h"
17#include "vgpu/gm20b/vgpu_gr_gm20b.h" 18#include "vgpu/gm20b/vgpu_gr_gm20b.h"
diff --git a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
index ea81cefe..b12f8a53 100644
--- a/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
+++ b/drivers/gpu/nvgpu/vgpu/mm_vgpu.c
@@ -16,6 +16,7 @@
16#include <linux/dma-mapping.h> 16#include <linux/dma-mapping.h>
17 17
18#include <nvgpu/kmem.h> 18#include <nvgpu/kmem.h>
19#include <nvgpu/dma.h>
19 20
20#include "vgpu/vgpu.h" 21#include "vgpu/vgpu.h"
21#include "gk20a/mm_gk20a.h" 22#include "gk20a/mm_gk20a.h"
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-02 20:34:26 -0400