diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/vm.c')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vm.c | 296 |
1 files changed, 296 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/vm.c b/drivers/gpu/nvgpu/common/linux/vm.c new file mode 100644 index 00000000..8e464627 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/vm.c | |||
@@ -0,0 +1,296 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify it | ||
5 | * under the terms and conditions of the GNU General Public License, | ||
6 | * version 2, as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | * more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
15 | */ | ||
16 | |||
17 | #include <linux/dma-buf.h> | ||
18 | #include <linux/scatterlist.h> | ||
19 | #include <uapi/linux/nvgpu.h> | ||
20 | |||
21 | #include <nvgpu/log.h> | ||
22 | #include <nvgpu/lock.h> | ||
23 | #include <nvgpu/rbtree.h> | ||
24 | #include <nvgpu/vm_area.h> | ||
25 | #include <nvgpu/nvgpu_mem.h> | ||
26 | #include <nvgpu/page_allocator.h> | ||
27 | #include <nvgpu/vidmem.h> | ||
28 | |||
29 | #include <nvgpu/linux/vm.h> | ||
30 | #include <nvgpu/linux/vidmem.h> | ||
31 | #include <nvgpu/linux/nvgpu_mem.h> | ||
32 | |||
33 | #include "gk20a/gk20a.h" | ||
34 | #include "gk20a/mm_gk20a.h" | ||
35 | |||
36 | #include "platform_gk20a.h" | ||
37 | #include "os_linux.h" | ||
38 | #include "dmabuf.h" | ||
39 | |||
40 | static struct nvgpu_mapped_buf *__nvgpu_vm_find_mapped_buf_reverse( | ||
41 | struct vm_gk20a *vm, struct dma_buf *dmabuf, u32 kind) | ||
42 | { | ||
43 | struct nvgpu_rbtree_node *node = NULL; | ||
44 | struct nvgpu_rbtree_node *root = vm->mapped_buffers; | ||
45 | |||
46 | nvgpu_rbtree_enum_start(0, &node, root); | ||
47 | |||
48 | while (node) { | ||
49 | struct nvgpu_mapped_buf *mapped_buffer = | ||
50 | mapped_buffer_from_rbtree_node(node); | ||
51 | |||
52 | if (mapped_buffer->os_priv.dmabuf == dmabuf && | ||
53 | mapped_buffer->kind == kind) | ||
54 | return mapped_buffer; | ||
55 | |||
56 | nvgpu_rbtree_enum_next(&node, node); | ||
57 | } | ||
58 | |||
59 | return NULL; | ||
60 | } | ||
61 | |||
62 | int nvgpu_vm_find_buf(struct vm_gk20a *vm, u64 gpu_va, | ||
63 | struct dma_buf **dmabuf, | ||
64 | u64 *offset) | ||
65 | { | ||
66 | struct nvgpu_mapped_buf *mapped_buffer; | ||
67 | |||
68 | gk20a_dbg_fn("gpu_va=0x%llx", gpu_va); | ||
69 | |||
70 | nvgpu_mutex_acquire(&vm->update_gmmu_lock); | ||
71 | |||
72 | mapped_buffer = __nvgpu_vm_find_mapped_buf_range(vm, gpu_va); | ||
73 | if (!mapped_buffer) { | ||
74 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
75 | return -EINVAL; | ||
76 | } | ||
77 | |||
78 | *dmabuf = mapped_buffer->os_priv.dmabuf; | ||
79 | *offset = gpu_va - mapped_buffer->addr; | ||
80 | |||
81 | nvgpu_mutex_release(&vm->update_gmmu_lock); | ||
82 | |||
83 | return 0; | ||
84 | } | ||
85 | |||
86 | u64 nvgpu_os_buf_get_size(struct nvgpu_os_buffer *os_buf) | ||
87 | { | ||
88 | return os_buf->dmabuf->size; | ||
89 | } | ||
90 | |||
91 | /* | ||
92 | * vm->update_gmmu_lock must be held. This checks to see if we already have | ||
93 | * mapped the passed buffer into this VM. If so, just return the existing | ||
94 | * mapping address. | ||
95 | */ | ||
96 | struct nvgpu_mapped_buf *nvgpu_vm_find_mapping(struct vm_gk20a *vm, | ||
97 | struct nvgpu_os_buffer *os_buf, | ||
98 | u64 map_addr, | ||
99 | u32 flags, | ||
100 | int kind) | ||
101 | { | ||
102 | struct gk20a *g = gk20a_from_vm(vm); | ||
103 | struct nvgpu_mapped_buf *mapped_buffer = NULL; | ||
104 | |||
105 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) { | ||
106 | mapped_buffer = __nvgpu_vm_find_mapped_buf(vm, map_addr); | ||
107 | if (!mapped_buffer) | ||
108 | return NULL; | ||
109 | |||
110 | if (mapped_buffer->os_priv.dmabuf != os_buf->dmabuf || | ||
111 | mapped_buffer->kind != (u32)kind) | ||
112 | return NULL; | ||
113 | } else { | ||
114 | mapped_buffer = | ||
115 | __nvgpu_vm_find_mapped_buf_reverse(vm, | ||
116 | os_buf->dmabuf, | ||
117 | kind); | ||
118 | if (!mapped_buffer) | ||
119 | return NULL; | ||
120 | } | ||
121 | |||
122 | if (mapped_buffer->flags != flags) | ||
123 | return NULL; | ||
124 | |||
125 | /* | ||
126 | * If we find the mapping here then that means we have mapped it already | ||
127 | * and the prior pin and get must be undone. | ||
128 | */ | ||
129 | gk20a_mm_unpin(os_buf->dev, os_buf->dmabuf, mapped_buffer->os_priv.sgt); | ||
130 | dma_buf_put(os_buf->dmabuf); | ||
131 | |||
132 | nvgpu_log(g, gpu_dbg_map, | ||
133 | "gv: 0x%04x_%08x + 0x%-7zu " | ||
134 | "[dma: 0x%010llx, pa: 0x%010llx] " | ||
135 | "pgsz=%-3dKb as=%-2d " | ||
136 | "flags=0x%x apt=%s (reused)", | ||
137 | u64_hi32(mapped_buffer->addr), u64_lo32(mapped_buffer->addr), | ||
138 | os_buf->dmabuf->size, | ||
139 | (u64)sg_dma_address(mapped_buffer->os_priv.sgt->sgl), | ||
140 | (u64)sg_phys(mapped_buffer->os_priv.sgt->sgl), | ||
141 | vm->gmmu_page_sizes[mapped_buffer->pgsz_idx] >> 10, | ||
142 | vm_aspace_id(vm), | ||
143 | mapped_buffer->flags, | ||
144 | nvgpu_aperture_str(gk20a_dmabuf_aperture(g, os_buf->dmabuf))); | ||
145 | |||
146 | return mapped_buffer; | ||
147 | } | ||
148 | |||
149 | int nvgpu_vm_map_linux(struct vm_gk20a *vm, | ||
150 | struct dma_buf *dmabuf, | ||
151 | u64 offset_align, | ||
152 | u32 flags, | ||
153 | s16 compr_kind, | ||
154 | s16 incompr_kind, | ||
155 | int rw_flag, | ||
156 | u64 buffer_offset, | ||
157 | u64 mapping_size, | ||
158 | struct vm_gk20a_mapping_batch *batch, | ||
159 | u64 *gpu_va) | ||
160 | { | ||
161 | struct gk20a *g = gk20a_from_vm(vm); | ||
162 | struct device *dev = dev_from_gk20a(g); | ||
163 | struct nvgpu_os_buffer os_buf = { dmabuf, dev }; | ||
164 | struct sg_table *sgt; | ||
165 | struct nvgpu_sgt *nvgpu_sgt = NULL; | ||
166 | struct nvgpu_mapped_buf *mapped_buffer = NULL; | ||
167 | u64 map_addr = 0ULL; | ||
168 | int err = 0; | ||
169 | |||
170 | if (flags & NVGPU_AS_MAP_BUFFER_FLAGS_FIXED_OFFSET) | ||
171 | map_addr = offset_align; | ||
172 | |||
173 | sgt = gk20a_mm_pin(dev, dmabuf); | ||
174 | if (IS_ERR(sgt)) { | ||
175 | nvgpu_warn(g, "Failed to pin dma_buf!"); | ||
176 | return PTR_ERR(sgt); | ||
177 | } | ||
178 | |||
179 | if (gk20a_dmabuf_aperture(g, dmabuf) == APERTURE_INVALID) { | ||
180 | err = -EINVAL; | ||
181 | goto clean_up; | ||
182 | } | ||
183 | |||
184 | nvgpu_sgt = nvgpu_linux_sgt_create(g, sgt); | ||
185 | if (!nvgpu_sgt) { | ||
186 | err = -ENOMEM; | ||
187 | goto clean_up; | ||
188 | } | ||
189 | |||
190 | mapped_buffer = nvgpu_vm_map(vm, | ||
191 | &os_buf, | ||
192 | nvgpu_sgt, | ||
193 | map_addr, | ||
194 | mapping_size, | ||
195 | buffer_offset, | ||
196 | rw_flag, | ||
197 | flags, | ||
198 | compr_kind, | ||
199 | incompr_kind, | ||
200 | batch, | ||
201 | gk20a_dmabuf_aperture(g, dmabuf)); | ||
202 | |||
203 | nvgpu_sgt_free(g, nvgpu_sgt); | ||
204 | |||
205 | if (IS_ERR(mapped_buffer)) { | ||
206 | err = PTR_ERR(mapped_buffer); | ||
207 | goto clean_up; | ||
208 | } | ||
209 | |||
210 | mapped_buffer->os_priv.dmabuf = dmabuf; | ||
211 | mapped_buffer->os_priv.sgt = sgt; | ||
212 | |||
213 | *gpu_va = mapped_buffer->addr; | ||
214 | return 0; | ||
215 | |||
216 | clean_up: | ||
217 | gk20a_mm_unpin(dev, dmabuf, sgt); | ||
218 | |||
219 | return err; | ||
220 | } | ||
221 | |||
222 | int nvgpu_vm_map_buffer(struct vm_gk20a *vm, | ||
223 | int dmabuf_fd, | ||
224 | u64 *offset_align, | ||
225 | u32 flags, /*NVGPU_AS_MAP_BUFFER_FLAGS_*/ | ||
226 | s16 compr_kind, | ||
227 | s16 incompr_kind, | ||
228 | u64 buffer_offset, | ||
229 | u64 mapping_size, | ||
230 | struct vm_gk20a_mapping_batch *batch) | ||
231 | { | ||
232 | int err = 0; | ||
233 | struct dma_buf *dmabuf; | ||
234 | u64 ret_va; | ||
235 | |||
236 | /* get ref to the mem handle (released on unmap_locked) */ | ||
237 | dmabuf = dma_buf_get(dmabuf_fd); | ||
238 | if (IS_ERR(dmabuf)) { | ||
239 | nvgpu_warn(gk20a_from_vm(vm), "%s: fd %d is not a dmabuf", | ||
240 | __func__, dmabuf_fd); | ||
241 | return PTR_ERR(dmabuf); | ||
242 | } | ||
243 | |||
244 | /* verify that we're not overflowing the buffer, i.e. | ||
245 | * (buffer_offset + mapping_size)> dmabuf->size. | ||
246 | * | ||
247 | * Since buffer_offset + mapping_size could overflow, first check | ||
248 | * that mapping size < dmabuf_size, at which point we can subtract | ||
249 | * mapping_size from both sides for the final comparison. | ||
250 | */ | ||
251 | if ((mapping_size > dmabuf->size) || | ||
252 | (buffer_offset > (dmabuf->size - mapping_size))) { | ||
253 | nvgpu_err(gk20a_from_vm(vm), | ||
254 | "buf size %llx < (offset(%llx) + map_size(%llx))\n", | ||
255 | (u64)dmabuf->size, buffer_offset, mapping_size); | ||
256 | return -EINVAL; | ||
257 | } | ||
258 | |||
259 | err = gk20a_dmabuf_alloc_drvdata(dmabuf, dev_from_vm(vm)); | ||
260 | if (err) { | ||
261 | dma_buf_put(dmabuf); | ||
262 | return err; | ||
263 | } | ||
264 | |||
265 | err = nvgpu_vm_map_linux(vm, dmabuf, *offset_align, | ||
266 | flags, compr_kind, incompr_kind, | ||
267 | gk20a_mem_flag_none, | ||
268 | buffer_offset, | ||
269 | mapping_size, | ||
270 | batch, | ||
271 | &ret_va); | ||
272 | |||
273 | if (!err) | ||
274 | *offset_align = ret_va; | ||
275 | else | ||
276 | dma_buf_put(dmabuf); | ||
277 | |||
278 | return err; | ||
279 | } | ||
280 | |||
281 | /* | ||
282 | * This is the function call-back for freeing OS specific components of an | ||
283 | * nvgpu_mapped_buf. This should most likely never be called outside of the | ||
284 | * core MM framework! | ||
285 | * | ||
286 | * Note: the VM lock will be held. | ||
287 | */ | ||
288 | void nvgpu_vm_unmap_system(struct nvgpu_mapped_buf *mapped_buffer) | ||
289 | { | ||
290 | struct vm_gk20a *vm = mapped_buffer->vm; | ||
291 | |||
292 | gk20a_mm_unpin(dev_from_vm(vm), mapped_buffer->os_priv.dmabuf, | ||
293 | mapped_buffer->os_priv.sgt); | ||
294 | |||
295 | dma_buf_put(mapped_buffer->os_priv.dmabuf); | ||
296 | } | ||