diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/dma.c | 415 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/semaphore.c | 1 |
2 files changed, 416 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/dma.c b/drivers/gpu/nvgpu/common/linux/dma.c new file mode 100644 index 00000000..755848ea --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/dma.c | |||
@@ -0,0 +1,415 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #include <linux/dma-attrs.h> | ||
18 | #include <linux/dma-mapping.h> | ||
19 | |||
20 | #include <nvgpu/dma.h> | ||
21 | #include <nvgpu/lock.h> | ||
22 | |||
23 | #include "gk20a/gk20a.h" | ||
24 | |||
25 | #if defined(CONFIG_GK20A_VIDMEM) | ||
26 | static u64 __gk20a_gmmu_alloc(struct nvgpu_allocator *allocator, dma_addr_t at, | ||
27 | size_t size) | ||
28 | { | ||
29 | u64 addr = 0; | ||
30 | |||
31 | if (at) | ||
32 | addr = nvgpu_alloc_fixed(allocator, at, size, 0); | ||
33 | else | ||
34 | addr = nvgpu_alloc(allocator, size); | ||
35 | |||
36 | return addr; | ||
37 | } | ||
38 | #endif | ||
39 | |||
40 | #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0) | ||
41 | static void gk20a_dma_flags_to_attrs(unsigned long *attrs, | ||
42 | unsigned long flags) | ||
43 | #define ATTR_ARG(x) *x | ||
44 | #else | ||
45 | static void gk20a_dma_flags_to_attrs(struct dma_attrs *attrs, | ||
46 | unsigned long flags) | ||
47 | #define ATTR_ARG(x) x | ||
48 | #endif | ||
49 | { | ||
50 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) | ||
51 | dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, ATTR_ARG(attrs)); | ||
52 | if (flags & NVGPU_DMA_FORCE_CONTIGUOUS) | ||
53 | dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, ATTR_ARG(attrs)); | ||
54 | if (flags & NVGPU_DMA_READ_ONLY) | ||
55 | dma_set_attr(DMA_ATTR_READ_ONLY, ATTR_ARG(attrs)); | ||
56 | #undef ATTR_ARG | ||
57 | } | ||
58 | |||
59 | int gk20a_gmmu_alloc(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
60 | { | ||
61 | return gk20a_gmmu_alloc_flags(g, 0, size, mem); | ||
62 | } | ||
63 | |||
64 | int gk20a_gmmu_alloc_flags(struct gk20a *g, unsigned long flags, size_t size, | ||
65 | struct nvgpu_mem *mem) | ||
66 | { | ||
67 | if (g->mm.vidmem_is_vidmem) { | ||
68 | /* | ||
69 | * Force the no-kernel-mapping flag on because we don't support | ||
70 | * the lack of it for vidmem - the user should not care when | ||
71 | * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a | ||
72 | * difference, the user should use the flag explicitly anyway. | ||
73 | */ | ||
74 | int err = gk20a_gmmu_alloc_flags_vid(g, | ||
75 | flags | NVGPU_DMA_NO_KERNEL_MAPPING, | ||
76 | size, mem); | ||
77 | |||
78 | if (!err) | ||
79 | return 0; | ||
80 | /* | ||
81 | * Fall back to sysmem (which may then also fail) in case | ||
82 | * vidmem is exhausted. | ||
83 | */ | ||
84 | } | ||
85 | |||
86 | return gk20a_gmmu_alloc_flags_sys(g, flags, size, mem); | ||
87 | } | ||
88 | |||
89 | int gk20a_gmmu_alloc_sys(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
90 | { | ||
91 | return gk20a_gmmu_alloc_flags_sys(g, 0, size, mem); | ||
92 | } | ||
93 | |||
94 | int gk20a_gmmu_alloc_flags_sys(struct gk20a *g, unsigned long flags, | ||
95 | size_t size, struct nvgpu_mem *mem) | ||
96 | { | ||
97 | struct device *d = dev_from_gk20a(g); | ||
98 | int err; | ||
99 | dma_addr_t iova; | ||
100 | |||
101 | gk20a_dbg_fn(""); | ||
102 | |||
103 | if (flags) { | ||
104 | DEFINE_DMA_ATTRS(dma_attrs); | ||
105 | |||
106 | gk20a_dma_flags_to_attrs(&dma_attrs, flags); | ||
107 | |||
108 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) { | ||
109 | mem->pages = dma_alloc_attrs(d, | ||
110 | size, &iova, GFP_KERNEL, | ||
111 | __DMA_ATTR(dma_attrs)); | ||
112 | if (!mem->pages) | ||
113 | return -ENOMEM; | ||
114 | } else { | ||
115 | mem->cpu_va = dma_alloc_attrs(d, | ||
116 | size, &iova, GFP_KERNEL, | ||
117 | __DMA_ATTR(dma_attrs)); | ||
118 | if (!mem->cpu_va) | ||
119 | return -ENOMEM; | ||
120 | } | ||
121 | } else { | ||
122 | mem->cpu_va = dma_alloc_coherent(d, size, &iova, GFP_KERNEL); | ||
123 | if (!mem->cpu_va) | ||
124 | return -ENOMEM; | ||
125 | } | ||
126 | |||
127 | if (flags & NVGPU_DMA_NO_KERNEL_MAPPING) | ||
128 | err = gk20a_get_sgtable_from_pages(d, &mem->sgt, mem->pages, | ||
129 | iova, size); | ||
130 | else { | ||
131 | err = gk20a_get_sgtable(d, &mem->sgt, mem->cpu_va, iova, size); | ||
132 | memset(mem->cpu_va, 0, size); | ||
133 | } | ||
134 | if (err) | ||
135 | goto fail_free; | ||
136 | |||
137 | mem->size = size; | ||
138 | mem->aperture = APERTURE_SYSMEM; | ||
139 | mem->flags = flags; | ||
140 | |||
141 | gk20a_dbg_fn("done"); | ||
142 | |||
143 | return 0; | ||
144 | |||
145 | fail_free: | ||
146 | dma_free_coherent(d, size, mem->cpu_va, iova); | ||
147 | mem->cpu_va = NULL; | ||
148 | mem->sgt = NULL; | ||
149 | return err; | ||
150 | } | ||
151 | |||
152 | int gk20a_gmmu_alloc_vid(struct gk20a *g, size_t size, struct nvgpu_mem *mem) | ||
153 | { | ||
154 | return gk20a_gmmu_alloc_flags_vid(g, | ||
155 | NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); | ||
156 | } | ||
157 | |||
158 | int gk20a_gmmu_alloc_flags_vid(struct gk20a *g, unsigned long flags, | ||
159 | size_t size, struct nvgpu_mem *mem) | ||
160 | { | ||
161 | return gk20a_gmmu_alloc_flags_vid_at(g, flags, size, mem, 0); | ||
162 | } | ||
163 | |||
164 | int gk20a_gmmu_alloc_flags_vid_at(struct gk20a *g, unsigned long flags, | ||
165 | size_t size, struct nvgpu_mem *mem, dma_addr_t at) | ||
166 | { | ||
167 | #if defined(CONFIG_GK20A_VIDMEM) | ||
168 | u64 addr; | ||
169 | int err; | ||
170 | struct nvgpu_allocator *vidmem_alloc = g->mm.vidmem.cleared ? | ||
171 | &g->mm.vidmem.allocator : | ||
172 | &g->mm.vidmem.bootstrap_allocator; | ||
173 | int before_pending; | ||
174 | |||
175 | gk20a_dbg_fn(""); | ||
176 | |||
177 | if (!nvgpu_alloc_initialized(&g->mm.vidmem.allocator)) | ||
178 | return -ENOSYS; | ||
179 | |||
180 | /* | ||
181 | * Our own allocator doesn't have any flags yet, and we can't | ||
182 | * kernel-map these, so require explicit flags. | ||
183 | */ | ||
184 | WARN_ON(flags != NVGPU_DMA_NO_KERNEL_MAPPING); | ||
185 | |||
186 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | ||
187 | before_pending = atomic64_read(&g->mm.vidmem.bytes_pending); | ||
188 | addr = __gk20a_gmmu_alloc(vidmem_alloc, at, size); | ||
189 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | ||
190 | if (!addr) { | ||
191 | /* | ||
192 | * If memory is known to be freed soon, let the user know that | ||
193 | * it may be available after a while. | ||
194 | */ | ||
195 | if (before_pending) | ||
196 | return -EAGAIN; | ||
197 | else | ||
198 | return -ENOMEM; | ||
199 | } | ||
200 | |||
201 | if (at) | ||
202 | mem->fixed = true; | ||
203 | else | ||
204 | mem->fixed = false; | ||
205 | |||
206 | mem->sgt = nvgpu_kzalloc(g, sizeof(struct sg_table)); | ||
207 | if (!mem->sgt) { | ||
208 | err = -ENOMEM; | ||
209 | goto fail_physfree; | ||
210 | } | ||
211 | |||
212 | err = sg_alloc_table(mem->sgt, 1, GFP_KERNEL); | ||
213 | if (err) | ||
214 | goto fail_kfree; | ||
215 | |||
216 | set_vidmem_page_alloc(mem->sgt->sgl, addr); | ||
217 | sg_set_page(mem->sgt->sgl, NULL, size, 0); | ||
218 | |||
219 | mem->size = size; | ||
220 | mem->aperture = APERTURE_VIDMEM; | ||
221 | mem->allocator = vidmem_alloc; | ||
222 | mem->flags = flags; | ||
223 | |||
224 | nvgpu_init_list_node(&mem->clear_list_entry); | ||
225 | |||
226 | gk20a_dbg_fn("done at 0x%llx size %zu", addr, size); | ||
227 | |||
228 | return 0; | ||
229 | |||
230 | fail_kfree: | ||
231 | nvgpu_kfree(g, mem->sgt); | ||
232 | fail_physfree: | ||
233 | nvgpu_free(&g->mm.vidmem.allocator, addr); | ||
234 | return err; | ||
235 | #else | ||
236 | return -ENOSYS; | ||
237 | #endif | ||
238 | } | ||
239 | |||
240 | int gk20a_gmmu_alloc_map(struct vm_gk20a *vm, size_t size, | ||
241 | struct nvgpu_mem *mem) | ||
242 | { | ||
243 | return gk20a_gmmu_alloc_map_flags(vm, 0, size, mem); | ||
244 | } | ||
245 | |||
246 | int gk20a_gmmu_alloc_map_flags(struct vm_gk20a *vm, unsigned long flags, | ||
247 | size_t size, struct nvgpu_mem *mem) | ||
248 | { | ||
249 | if (vm->mm->vidmem_is_vidmem) { | ||
250 | /* | ||
251 | * Force the no-kernel-mapping flag on because we don't support | ||
252 | * the lack of it for vidmem - the user should not care when | ||
253 | * using gk20a_gmmu_alloc_map and it's vidmem, or if there's a | ||
254 | * difference, the user should use the flag explicitly anyway. | ||
255 | */ | ||
256 | int err = gk20a_gmmu_alloc_map_flags_vid(vm, | ||
257 | flags | NVGPU_DMA_NO_KERNEL_MAPPING, | ||
258 | size, mem); | ||
259 | |||
260 | if (!err) | ||
261 | return 0; | ||
262 | /* | ||
263 | * Fall back to sysmem (which may then also fail) in case | ||
264 | * vidmem is exhausted. | ||
265 | */ | ||
266 | } | ||
267 | |||
268 | return gk20a_gmmu_alloc_map_flags_sys(vm, flags, size, mem); | ||
269 | } | ||
270 | |||
271 | int gk20a_gmmu_alloc_map_sys(struct vm_gk20a *vm, size_t size, | ||
272 | struct nvgpu_mem *mem) | ||
273 | { | ||
274 | return gk20a_gmmu_alloc_map_flags_sys(vm, 0, size, mem); | ||
275 | } | ||
276 | |||
277 | int gk20a_gmmu_alloc_map_flags_sys(struct vm_gk20a *vm, unsigned long flags, | ||
278 | size_t size, struct nvgpu_mem *mem) | ||
279 | { | ||
280 | int err = gk20a_gmmu_alloc_flags_sys(vm->mm->g, flags, size, mem); | ||
281 | |||
282 | if (err) | ||
283 | return err; | ||
284 | |||
285 | mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0, | ||
286 | gk20a_mem_flag_none, false, | ||
287 | mem->aperture); | ||
288 | if (!mem->gpu_va) { | ||
289 | err = -ENOMEM; | ||
290 | goto fail_free; | ||
291 | } | ||
292 | |||
293 | return 0; | ||
294 | |||
295 | fail_free: | ||
296 | gk20a_gmmu_free(vm->mm->g, mem); | ||
297 | return err; | ||
298 | } | ||
299 | |||
300 | int gk20a_gmmu_alloc_map_vid(struct vm_gk20a *vm, size_t size, | ||
301 | struct nvgpu_mem *mem) | ||
302 | { | ||
303 | return gk20a_gmmu_alloc_map_flags_vid(vm, | ||
304 | NVGPU_DMA_NO_KERNEL_MAPPING, size, mem); | ||
305 | } | ||
306 | |||
307 | int gk20a_gmmu_alloc_map_flags_vid(struct vm_gk20a *vm, unsigned long flags, | ||
308 | size_t size, struct nvgpu_mem *mem) | ||
309 | { | ||
310 | int err = gk20a_gmmu_alloc_flags_vid(vm->mm->g, flags, size, mem); | ||
311 | |||
312 | if (err) | ||
313 | return err; | ||
314 | |||
315 | mem->gpu_va = gk20a_gmmu_map(vm, &mem->sgt, size, 0, | ||
316 | gk20a_mem_flag_none, false, | ||
317 | mem->aperture); | ||
318 | if (!mem->gpu_va) { | ||
319 | err = -ENOMEM; | ||
320 | goto fail_free; | ||
321 | } | ||
322 | |||
323 | return 0; | ||
324 | |||
325 | fail_free: | ||
326 | gk20a_gmmu_free(vm->mm->g, mem); | ||
327 | return err; | ||
328 | } | ||
329 | |||
330 | static void gk20a_gmmu_free_sys(struct gk20a *g, struct nvgpu_mem *mem) | ||
331 | { | ||
332 | struct device *d = dev_from_gk20a(g); | ||
333 | |||
334 | if (mem->cpu_va || mem->pages) { | ||
335 | if (mem->flags) { | ||
336 | DEFINE_DMA_ATTRS(dma_attrs); | ||
337 | |||
338 | gk20a_dma_flags_to_attrs(&dma_attrs, mem->flags); | ||
339 | |||
340 | if (mem->flags & NVGPU_DMA_NO_KERNEL_MAPPING) { | ||
341 | dma_free_attrs(d, mem->size, mem->pages, | ||
342 | sg_dma_address(mem->sgt->sgl), | ||
343 | __DMA_ATTR(dma_attrs)); | ||
344 | } else { | ||
345 | dma_free_attrs(d, mem->size, mem->cpu_va, | ||
346 | sg_dma_address(mem->sgt->sgl), | ||
347 | __DMA_ATTR(dma_attrs)); | ||
348 | } | ||
349 | } else { | ||
350 | dma_free_coherent(d, mem->size, mem->cpu_va, | ||
351 | sg_dma_address(mem->sgt->sgl)); | ||
352 | } | ||
353 | mem->cpu_va = NULL; | ||
354 | mem->pages = NULL; | ||
355 | } | ||
356 | |||
357 | if (mem->sgt) | ||
358 | gk20a_free_sgtable(g, &mem->sgt); | ||
359 | |||
360 | mem->size = 0; | ||
361 | mem->aperture = APERTURE_INVALID; | ||
362 | } | ||
363 | |||
364 | static void gk20a_gmmu_free_vid(struct gk20a *g, struct nvgpu_mem *mem) | ||
365 | { | ||
366 | #if defined(CONFIG_GK20A_VIDMEM) | ||
367 | bool was_empty; | ||
368 | |||
369 | /* Sanity check - only this supported when allocating. */ | ||
370 | WARN_ON(mem->flags != NVGPU_DMA_NO_KERNEL_MAPPING); | ||
371 | |||
372 | if (mem->user_mem) { | ||
373 | nvgpu_mutex_acquire(&g->mm.vidmem.clear_list_mutex); | ||
374 | was_empty = nvgpu_list_empty(&g->mm.vidmem.clear_list_head); | ||
375 | nvgpu_list_add_tail(&mem->clear_list_entry, | ||
376 | &g->mm.vidmem.clear_list_head); | ||
377 | atomic64_add(mem->size, &g->mm.vidmem.bytes_pending); | ||
378 | nvgpu_mutex_release(&g->mm.vidmem.clear_list_mutex); | ||
379 | |||
380 | if (was_empty) { | ||
381 | cancel_work_sync(&g->mm.vidmem.clear_mem_worker); | ||
382 | schedule_work(&g->mm.vidmem.clear_mem_worker); | ||
383 | } | ||
384 | } else { | ||
385 | nvgpu_memset(g, mem, 0, 0, mem->size); | ||
386 | nvgpu_free(mem->allocator, | ||
387 | (u64)get_vidmem_page_alloc(mem->sgt->sgl)); | ||
388 | gk20a_free_sgtable(g, &mem->sgt); | ||
389 | |||
390 | mem->size = 0; | ||
391 | mem->aperture = APERTURE_INVALID; | ||
392 | } | ||
393 | #endif | ||
394 | } | ||
395 | |||
396 | void gk20a_gmmu_free(struct gk20a *g, struct nvgpu_mem *mem) | ||
397 | { | ||
398 | switch (mem->aperture) { | ||
399 | case APERTURE_SYSMEM: | ||
400 | return gk20a_gmmu_free_sys(g, mem); | ||
401 | case APERTURE_VIDMEM: | ||
402 | return gk20a_gmmu_free_vid(g, mem); | ||
403 | default: | ||
404 | break; /* like free() on "null" memory */ | ||
405 | } | ||
406 | } | ||
407 | |||
408 | void gk20a_gmmu_unmap_free(struct vm_gk20a *vm, struct nvgpu_mem *mem) | ||
409 | { | ||
410 | if (mem->gpu_va) | ||
411 | gk20a_gmmu_unmap(vm, mem->gpu_va, mem->size, gk20a_mem_flag_none); | ||
412 | mem->gpu_va = 0; | ||
413 | |||
414 | gk20a_gmmu_free(vm->mm->g, mem); | ||
415 | } | ||
diff --git a/drivers/gpu/nvgpu/common/semaphore.c b/drivers/gpu/nvgpu/common/semaphore.c index 6fb6c27e..cfe1149f 100644 --- a/drivers/gpu/nvgpu/common/semaphore.c +++ b/drivers/gpu/nvgpu/common/semaphore.c | |||
@@ -18,6 +18,7 @@ | |||
18 | #include <linux/dma-mapping.h> | 18 | #include <linux/dma-mapping.h> |
19 | #include <linux/highmem.h> | 19 | #include <linux/highmem.h> |
20 | 20 | ||
21 | #include <nvgpu/dma.h> | ||
21 | #include <nvgpu/semaphore.h> | 22 | #include <nvgpu/semaphore.h> |
22 | #include <nvgpu/kmem.h> | 23 | #include <nvgpu/kmem.h> |
23 | 24 | ||