diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/vgpu/mm_vgpu.c')
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/vgpu/mm_vgpu.c | 363 |
1 files changed, 363 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/vgpu/mm_vgpu.c b/drivers/gpu/nvgpu/common/linux/vgpu/mm_vgpu.c new file mode 100644 index 00000000..f8c5c406 --- /dev/null +++ b/drivers/gpu/nvgpu/common/linux/vgpu/mm_vgpu.c | |||
@@ -0,0 +1,363 @@ | |||
1 | /* | ||
2 | * Virtualized GPU Memory Management | ||
3 | * | ||
4 | * Copyright (c) 2014-2017, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify it | ||
7 | * under the terms and conditions of the GNU General Public License, | ||
8 | * version 2, as published by the Free Software Foundation. | ||
9 | * | ||
10 | * This program is distributed in the hope it will be useful, but WITHOUT | ||
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
13 | * more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU General Public License | ||
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
17 | */ | ||
18 | |||
19 | #include <linux/dma-mapping.h> | ||
20 | #include <uapi/linux/nvgpu.h> | ||
21 | |||
22 | #include <nvgpu/kmem.h> | ||
23 | #include <nvgpu/dma.h> | ||
24 | #include <nvgpu/bug.h> | ||
25 | #include <nvgpu/vm.h> | ||
26 | #include <nvgpu/vm_area.h> | ||
27 | |||
28 | #include <nvgpu/vgpu/vm.h> | ||
29 | |||
30 | #include <nvgpu/linux/vm.h> | ||
31 | #include <nvgpu/linux/nvgpu_mem.h> | ||
32 | |||
33 | #include "vgpu.h" | ||
34 | #include "mm_vgpu.h" | ||
35 | #include "gk20a/mm_gk20a.h" | ||
36 | #include "gm20b/mm_gm20b.h" | ||
37 | |||
38 | static int vgpu_init_mm_setup_sw(struct gk20a *g) | ||
39 | { | ||
40 | struct mm_gk20a *mm = &g->mm; | ||
41 | |||
42 | gk20a_dbg_fn(""); | ||
43 | |||
44 | if (mm->sw_ready) { | ||
45 | gk20a_dbg_fn("skip init"); | ||
46 | return 0; | ||
47 | } | ||
48 | |||
49 | nvgpu_mutex_init(&mm->tlb_lock); | ||
50 | nvgpu_mutex_init(&mm->priv_lock); | ||
51 | |||
52 | mm->g = g; | ||
53 | |||
54 | /*TBD: make channel vm size configurable */ | ||
55 | mm->channel.user_size = NV_MM_DEFAULT_USER_SIZE; | ||
56 | mm->channel.kernel_size = NV_MM_DEFAULT_KERNEL_SIZE; | ||
57 | |||
58 | gk20a_dbg_info("channel vm size: user %dMB kernel %dMB", | ||
59 | (int)(mm->channel.user_size >> 20), | ||
60 | (int)(mm->channel.kernel_size >> 20)); | ||
61 | |||
62 | mm->sw_ready = true; | ||
63 | |||
64 | return 0; | ||
65 | } | ||
66 | |||
67 | int vgpu_init_mm_support(struct gk20a *g) | ||
68 | { | ||
69 | int err; | ||
70 | |||
71 | gk20a_dbg_fn(""); | ||
72 | |||
73 | err = vgpu_init_mm_setup_sw(g); | ||
74 | if (err) | ||
75 | return err; | ||
76 | |||
77 | if (g->ops.mm.init_mm_setup_hw) | ||
78 | err = g->ops.mm.init_mm_setup_hw(g); | ||
79 | |||
80 | return err; | ||
81 | } | ||
82 | |||
83 | u64 vgpu_locked_gmmu_map(struct vm_gk20a *vm, | ||
84 | u64 map_offset, | ||
85 | struct nvgpu_sgt *sgt, | ||
86 | u64 buffer_offset, | ||
87 | u64 size, | ||
88 | int pgsz_idx, | ||
89 | u8 kind_v, | ||
90 | u32 ctag_offset, | ||
91 | u32 flags, | ||
92 | int rw_flag, | ||
93 | bool clear_ctags, | ||
94 | bool sparse, | ||
95 | bool priv, | ||
96 | struct vm_gk20a_mapping_batch *batch, | ||
97 | enum nvgpu_aperture aperture) | ||
98 | { | ||
99 | int err = 0; | ||
100 | struct device *d = dev_from_vm(vm); | ||
101 | struct gk20a *g = gk20a_from_vm(vm); | ||
102 | struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(d); | ||
103 | struct tegra_vgpu_cmd_msg msg; | ||
104 | struct tegra_vgpu_as_map_params *p = &msg.params.as_map; | ||
105 | u64 addr = nvgpu_sgt_get_gpu_addr(g, sgt, sgt->sgl, NULL); | ||
106 | u8 prot; | ||
107 | |||
108 | gk20a_dbg_fn(""); | ||
109 | |||
110 | /* Allocate (or validate when map_offset != 0) the virtual address. */ | ||
111 | if (!map_offset) { | ||
112 | map_offset = __nvgpu_vm_alloc_va(vm, size, | ||
113 | pgsz_idx); | ||
114 | if (!map_offset) { | ||
115 | nvgpu_err(g, "failed to allocate va space"); | ||
116 | err = -ENOMEM; | ||
117 | goto fail; | ||
118 | } | ||
119 | } | ||
120 | |||
121 | if (rw_flag == gk20a_mem_flag_read_only) | ||
122 | prot = TEGRA_VGPU_MAP_PROT_READ_ONLY; | ||
123 | else if (rw_flag == gk20a_mem_flag_write_only) | ||
124 | prot = TEGRA_VGPU_MAP_PROT_WRITE_ONLY; | ||
125 | else | ||
126 | prot = TEGRA_VGPU_MAP_PROT_NONE; | ||
127 | |||
128 | msg.cmd = TEGRA_VGPU_CMD_AS_MAP; | ||
129 | msg.handle = vgpu_get_handle(g); | ||
130 | p->handle = vm->handle; | ||
131 | p->addr = addr; | ||
132 | p->gpu_va = map_offset; | ||
133 | p->size = size; | ||
134 | if (pgsz_idx == gmmu_page_size_kernel) { | ||
135 | u32 page_size = vm->gmmu_page_sizes[pgsz_idx]; | ||
136 | |||
137 | if (page_size == vm->gmmu_page_sizes[gmmu_page_size_small]) { | ||
138 | pgsz_idx = gmmu_page_size_small; | ||
139 | } else if (page_size == | ||
140 | vm->gmmu_page_sizes[gmmu_page_size_big]) { | ||
141 | pgsz_idx = gmmu_page_size_big; | ||
142 | } else { | ||
143 | nvgpu_err(g, "invalid kernel page size %d", | ||
144 | page_size); | ||
145 | goto fail; | ||
146 | } | ||
147 | } | ||
148 | p->pgsz_idx = pgsz_idx; | ||
149 | p->iova = mapping ? 1 : 0; | ||
150 | p->kind = kind_v; | ||
151 | p->cacheable = (flags & NVGPU_AS_MAP_BUFFER_FLAGS_CACHEABLE) ? 1 : 0; | ||
152 | p->prot = prot; | ||
153 | p->ctag_offset = ctag_offset; | ||
154 | p->clear_ctags = clear_ctags; | ||
155 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
156 | err = err ? err : msg.ret; | ||
157 | if (err) | ||
158 | goto fail; | ||
159 | |||
160 | /* TLB invalidate handled on server side */ | ||
161 | |||
162 | return map_offset; | ||
163 | fail: | ||
164 | nvgpu_err(g, "%s: failed with err=%d", __func__, err); | ||
165 | return 0; | ||
166 | } | ||
167 | |||
168 | void vgpu_locked_gmmu_unmap(struct vm_gk20a *vm, | ||
169 | u64 vaddr, | ||
170 | u64 size, | ||
171 | int pgsz_idx, | ||
172 | bool va_allocated, | ||
173 | int rw_flag, | ||
174 | bool sparse, | ||
175 | struct vm_gk20a_mapping_batch *batch) | ||
176 | { | ||
177 | struct gk20a *g = gk20a_from_vm(vm); | ||
178 | struct tegra_vgpu_cmd_msg msg; | ||
179 | struct tegra_vgpu_as_map_params *p = &msg.params.as_map; | ||
180 | int err; | ||
181 | |||
182 | gk20a_dbg_fn(""); | ||
183 | |||
184 | if (va_allocated) { | ||
185 | err = __nvgpu_vm_free_va(vm, vaddr, pgsz_idx); | ||
186 | if (err) { | ||
187 | dev_err(dev_from_vm(vm), | ||
188 | "failed to free va"); | ||
189 | return; | ||
190 | } | ||
191 | } | ||
192 | |||
193 | msg.cmd = TEGRA_VGPU_CMD_AS_UNMAP; | ||
194 | msg.handle = vgpu_get_handle(g); | ||
195 | p->handle = vm->handle; | ||
196 | p->gpu_va = vaddr; | ||
197 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
198 | if (err || msg.ret) | ||
199 | dev_err(dev_from_vm(vm), | ||
200 | "failed to update gmmu ptes on unmap"); | ||
201 | |||
202 | /* TLB invalidate handled on server side */ | ||
203 | } | ||
204 | |||
205 | /* | ||
206 | * This is called by the common VM init routine to handle vGPU specifics of | ||
207 | * intializing a VM on a vGPU. This alone is not enough to init a VM. See | ||
208 | * nvgpu_vm_init(). | ||
209 | */ | ||
210 | int vgpu_vm_init(struct gk20a *g, struct vm_gk20a *vm) | ||
211 | { | ||
212 | struct tegra_vgpu_cmd_msg msg; | ||
213 | struct tegra_vgpu_as_share_params *p = &msg.params.as_share; | ||
214 | int err; | ||
215 | |||
216 | msg.cmd = TEGRA_VGPU_CMD_AS_ALLOC_SHARE; | ||
217 | msg.handle = vgpu_get_handle(g); | ||
218 | p->size = vm->va_limit; | ||
219 | p->big_page_size = vm->big_page_size; | ||
220 | |||
221 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
222 | if (err || msg.ret) | ||
223 | return -ENOMEM; | ||
224 | |||
225 | vm->handle = p->handle; | ||
226 | |||
227 | return 0; | ||
228 | } | ||
229 | |||
230 | /* | ||
231 | * Similar to vgpu_vm_init() this is called as part of the cleanup path for | ||
232 | * VMs. This alone is not enough to remove a VM - see nvgpu_vm_remove(). | ||
233 | */ | ||
234 | void vgpu_vm_remove(struct vm_gk20a *vm) | ||
235 | { | ||
236 | struct gk20a *g = gk20a_from_vm(vm); | ||
237 | struct tegra_vgpu_cmd_msg msg; | ||
238 | struct tegra_vgpu_as_share_params *p = &msg.params.as_share; | ||
239 | int err; | ||
240 | |||
241 | msg.cmd = TEGRA_VGPU_CMD_AS_FREE_SHARE; | ||
242 | msg.handle = vgpu_get_handle(g); | ||
243 | p->handle = vm->handle; | ||
244 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
245 | WARN_ON(err || msg.ret); | ||
246 | } | ||
247 | |||
248 | u64 vgpu_bar1_map(struct gk20a *g, struct sg_table **sgt, u64 size) | ||
249 | { | ||
250 | struct dma_iommu_mapping *mapping = | ||
251 | to_dma_iommu_mapping(dev_from_gk20a(g)); | ||
252 | u64 addr = nvgpu_mem_get_addr_sgl(g, (*sgt)->sgl); | ||
253 | struct tegra_vgpu_cmd_msg msg; | ||
254 | struct tegra_vgpu_as_map_params *p = &msg.params.as_map; | ||
255 | int err; | ||
256 | |||
257 | msg.cmd = TEGRA_VGPU_CMD_MAP_BAR1; | ||
258 | msg.handle = vgpu_get_handle(g); | ||
259 | p->addr = addr; | ||
260 | p->size = size; | ||
261 | p->iova = mapping ? 1 : 0; | ||
262 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
263 | if (err || msg.ret) | ||
264 | addr = 0; | ||
265 | else | ||
266 | addr = p->gpu_va; | ||
267 | |||
268 | return addr; | ||
269 | } | ||
270 | |||
271 | int vgpu_vm_bind_channel(struct gk20a_as_share *as_share, | ||
272 | struct channel_gk20a *ch) | ||
273 | { | ||
274 | struct vm_gk20a *vm = as_share->vm; | ||
275 | struct tegra_vgpu_cmd_msg msg; | ||
276 | struct tegra_vgpu_as_bind_share_params *p = &msg.params.as_bind_share; | ||
277 | int err; | ||
278 | |||
279 | gk20a_dbg_fn(""); | ||
280 | |||
281 | ch->vm = vm; | ||
282 | msg.cmd = TEGRA_VGPU_CMD_AS_BIND_SHARE; | ||
283 | msg.handle = vgpu_get_handle(ch->g); | ||
284 | p->as_handle = vm->handle; | ||
285 | p->chan_handle = ch->virt_ctx; | ||
286 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
287 | |||
288 | if (err || msg.ret) { | ||
289 | ch->vm = NULL; | ||
290 | err = -ENOMEM; | ||
291 | } | ||
292 | |||
293 | if (ch->vm) | ||
294 | nvgpu_vm_get(ch->vm); | ||
295 | |||
296 | return err; | ||
297 | } | ||
298 | |||
299 | static void vgpu_cache_maint(u64 handle, u8 op) | ||
300 | { | ||
301 | struct tegra_vgpu_cmd_msg msg; | ||
302 | struct tegra_vgpu_cache_maint_params *p = &msg.params.cache_maint; | ||
303 | int err; | ||
304 | |||
305 | msg.cmd = TEGRA_VGPU_CMD_CACHE_MAINT; | ||
306 | msg.handle = handle; | ||
307 | p->op = op; | ||
308 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
309 | WARN_ON(err || msg.ret); | ||
310 | } | ||
311 | |||
312 | int vgpu_mm_fb_flush(struct gk20a *g) | ||
313 | { | ||
314 | |||
315 | gk20a_dbg_fn(""); | ||
316 | |||
317 | vgpu_cache_maint(vgpu_get_handle(g), TEGRA_VGPU_FB_FLUSH); | ||
318 | return 0; | ||
319 | } | ||
320 | |||
321 | void vgpu_mm_l2_invalidate(struct gk20a *g) | ||
322 | { | ||
323 | |||
324 | gk20a_dbg_fn(""); | ||
325 | |||
326 | vgpu_cache_maint(vgpu_get_handle(g), TEGRA_VGPU_L2_MAINT_INV); | ||
327 | } | ||
328 | |||
329 | void vgpu_mm_l2_flush(struct gk20a *g, bool invalidate) | ||
330 | { | ||
331 | u8 op; | ||
332 | |||
333 | gk20a_dbg_fn(""); | ||
334 | |||
335 | if (invalidate) | ||
336 | op = TEGRA_VGPU_L2_MAINT_FLUSH_INV; | ||
337 | else | ||
338 | op = TEGRA_VGPU_L2_MAINT_FLUSH; | ||
339 | |||
340 | vgpu_cache_maint(vgpu_get_handle(g), op); | ||
341 | } | ||
342 | |||
343 | void vgpu_mm_tlb_invalidate(struct gk20a *g, struct nvgpu_mem *pdb) | ||
344 | { | ||
345 | gk20a_dbg_fn(""); | ||
346 | |||
347 | nvgpu_err(g, "call to RM server not supported"); | ||
348 | } | ||
349 | |||
350 | void vgpu_mm_mmu_set_debug_mode(struct gk20a *g, bool enable) | ||
351 | { | ||
352 | struct tegra_vgpu_cmd_msg msg; | ||
353 | struct tegra_vgpu_mmu_debug_mode *p = &msg.params.mmu_debug_mode; | ||
354 | int err; | ||
355 | |||
356 | gk20a_dbg_fn(""); | ||
357 | |||
358 | msg.cmd = TEGRA_VGPU_CMD_SET_MMU_DEBUG_MODE; | ||
359 | msg.handle = vgpu_get_handle(g); | ||
360 | p->enable = (u32)enable; | ||
361 | err = vgpu_comm_sendrecv(&msg, sizeof(msg), sizeof(msg)); | ||
362 | WARN_ON(err || msg.ret); | ||
363 | } | ||