diff options
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm/mm.c')
-rw-r--r-- | drivers/gpu/nvgpu/common/mm/mm.c | 119 |
1 files changed, 79 insertions, 40 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/mm.c b/drivers/gpu/nvgpu/common/mm/mm.c index 2c3a1cd6..0608d66a 100644 --- a/drivers/gpu/nvgpu/common/mm/mm.c +++ b/drivers/gpu/nvgpu/common/mm/mm.c | |||
@@ -40,8 +40,9 @@ enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm, | |||
40 | struct nvgpu_vm_area *vm_area; | 40 | struct nvgpu_vm_area *vm_area; |
41 | 41 | ||
42 | vm_area = nvgpu_vm_area_find(vm, base); | 42 | vm_area = nvgpu_vm_area_find(vm, base); |
43 | if (!vm_area) | 43 | if (!vm_area) { |
44 | return gmmu_page_size_small; | 44 | return gmmu_page_size_small; |
45 | } | ||
45 | 46 | ||
46 | return vm_area->pgsz_idx; | 47 | return vm_area->pgsz_idx; |
47 | } | 48 | } |
@@ -53,14 +54,16 @@ static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm, | |||
53 | u64 base, u64 size) | 54 | u64 base, u64 size) |
54 | { | 55 | { |
55 | if (!base) { | 56 | if (!base) { |
56 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) | 57 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) { |
57 | return gmmu_page_size_big; | 58 | return gmmu_page_size_big; |
59 | } | ||
58 | return gmmu_page_size_small; | 60 | return gmmu_page_size_small; |
59 | } else { | 61 | } else { |
60 | if (base < __nv_gmmu_va_small_page_limit()) | 62 | if (base < __nv_gmmu_va_small_page_limit()) { |
61 | return gmmu_page_size_small; | 63 | return gmmu_page_size_small; |
62 | else | 64 | } else { |
63 | return gmmu_page_size_big; | 65 | return gmmu_page_size_big; |
66 | } | ||
64 | } | 67 | } |
65 | } | 68 | } |
66 | 69 | ||
@@ -89,18 +92,22 @@ enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size) | |||
89 | { | 92 | { |
90 | struct gk20a *g = gk20a_from_vm(vm); | 93 | struct gk20a *g = gk20a_from_vm(vm); |
91 | 94 | ||
92 | if (!vm->big_pages) | 95 | if (!vm->big_pages) { |
93 | return gmmu_page_size_small; | 96 | return gmmu_page_size_small; |
97 | } | ||
94 | 98 | ||
95 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) | 99 | if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) { |
96 | return __get_pte_size_split_addr(vm, base, size); | 100 | return __get_pte_size_split_addr(vm, base, size); |
101 | } | ||
97 | 102 | ||
98 | if (base) | 103 | if (base) { |
99 | return __get_pte_size_fixed_map(vm, base, size); | 104 | return __get_pte_size_fixed_map(vm, base, size); |
105 | } | ||
100 | 106 | ||
101 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && | 107 | if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && |
102 | nvgpu_iommuable(g)) | 108 | nvgpu_iommuable(g)) { |
103 | return gmmu_page_size_big; | 109 | return gmmu_page_size_big; |
110 | } | ||
104 | return gmmu_page_size_small; | 111 | return gmmu_page_size_small; |
105 | } | 112 | } |
106 | 113 | ||
@@ -137,8 +144,9 @@ u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block) | |||
137 | 144 | ||
138 | void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) | 145 | void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) |
139 | { | 146 | { |
140 | if (nvgpu_mem_is_valid(inst_block)) | 147 | if (nvgpu_mem_is_valid(inst_block)) { |
141 | nvgpu_dma_free(g, inst_block); | 148 | nvgpu_dma_free(g, inst_block); |
149 | } | ||
142 | } | 150 | } |
143 | 151 | ||
144 | static int nvgpu_alloc_sysmem_flush(struct gk20a *g) | 152 | static int nvgpu_alloc_sysmem_flush(struct gk20a *g) |
@@ -150,8 +158,9 @@ static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm) | |||
150 | { | 158 | { |
151 | struct gk20a *g = gk20a_from_mm(mm); | 159 | struct gk20a *g = gk20a_from_mm(mm); |
152 | 160 | ||
153 | if (mm->vidmem.ce_ctx_id != (u32)~0) | 161 | if (mm->vidmem.ce_ctx_id != (u32)~0) { |
154 | gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); | 162 | gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); |
163 | } | ||
155 | 164 | ||
156 | mm->vidmem.ce_ctx_id = (u32)~0; | 165 | mm->vidmem.ce_ctx_id = (u32)~0; |
157 | 166 | ||
@@ -162,11 +171,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm) | |||
162 | { | 171 | { |
163 | struct gk20a *g = gk20a_from_mm(mm); | 172 | struct gk20a *g = gk20a_from_mm(mm); |
164 | 173 | ||
165 | if (g->ops.mm.fault_info_mem_destroy) | 174 | if (g->ops.mm.fault_info_mem_destroy) { |
166 | g->ops.mm.fault_info_mem_destroy(g); | 175 | g->ops.mm.fault_info_mem_destroy(g); |
176 | } | ||
167 | 177 | ||
168 | if (g->ops.mm.remove_bar2_vm) | 178 | if (g->ops.mm.remove_bar2_vm) { |
169 | g->ops.mm.remove_bar2_vm(g); | 179 | g->ops.mm.remove_bar2_vm(g); |
180 | } | ||
170 | 181 | ||
171 | nvgpu_free_inst_block(g, &mm->bar1.inst_block); | 182 | nvgpu_free_inst_block(g, &mm->bar1.inst_block); |
172 | nvgpu_vm_put(mm->bar1.vm); | 183 | nvgpu_vm_put(mm->bar1.vm); |
@@ -175,8 +186,9 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm) | |||
175 | nvgpu_free_inst_block(g, &mm->hwpm.inst_block); | 186 | nvgpu_free_inst_block(g, &mm->hwpm.inst_block); |
176 | nvgpu_vm_put(mm->pmu.vm); | 187 | nvgpu_vm_put(mm->pmu.vm); |
177 | 188 | ||
178 | if (g->has_cde) | 189 | if (g->has_cde) { |
179 | nvgpu_vm_put(mm->cde.vm); | 190 | nvgpu_vm_put(mm->cde.vm); |
191 | } | ||
180 | 192 | ||
181 | nvgpu_semaphore_sea_destroy(g); | 193 | nvgpu_semaphore_sea_destroy(g); |
182 | nvgpu_vidmem_destroy(g); | 194 | nvgpu_vidmem_destroy(g); |
@@ -208,12 +220,14 @@ static int nvgpu_init_system_vm(struct mm_gk20a *mm) | |||
208 | true, | 220 | true, |
209 | false, | 221 | false, |
210 | "system"); | 222 | "system"); |
211 | if (!mm->pmu.vm) | 223 | if (!mm->pmu.vm) { |
212 | return -ENOMEM; | 224 | return -ENOMEM; |
225 | } | ||
213 | 226 | ||
214 | err = g->ops.mm.alloc_inst_block(g, inst_block); | 227 | err = g->ops.mm.alloc_inst_block(g, inst_block); |
215 | if (err) | 228 | if (err) { |
216 | goto clean_up_vm; | 229 | goto clean_up_vm; |
230 | } | ||
217 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); | 231 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); |
218 | 232 | ||
219 | return 0; | 233 | return 0; |
@@ -230,8 +244,9 @@ static int nvgpu_init_hwpm(struct mm_gk20a *mm) | |||
230 | struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; | 244 | struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; |
231 | 245 | ||
232 | err = g->ops.mm.alloc_inst_block(g, inst_block); | 246 | err = g->ops.mm.alloc_inst_block(g, inst_block); |
233 | if (err) | 247 | if (err) { |
234 | return err; | 248 | return err; |
249 | } | ||
235 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); | 250 | g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); |
236 | 251 | ||
237 | return 0; | 252 | return 0; |
@@ -247,8 +262,9 @@ static int nvgpu_init_cde_vm(struct mm_gk20a *mm) | |||
247 | NV_MM_DEFAULT_KERNEL_SIZE, | 262 | NV_MM_DEFAULT_KERNEL_SIZE, |
248 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, | 263 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, |
249 | false, false, "cde"); | 264 | false, false, "cde"); |
250 | if (!mm->cde.vm) | 265 | if (!mm->cde.vm) { |
251 | return -ENOMEM; | 266 | return -ENOMEM; |
267 | } | ||
252 | return 0; | 268 | return 0; |
253 | } | 269 | } |
254 | 270 | ||
@@ -262,8 +278,9 @@ static int nvgpu_init_ce_vm(struct mm_gk20a *mm) | |||
262 | NV_MM_DEFAULT_KERNEL_SIZE, | 278 | NV_MM_DEFAULT_KERNEL_SIZE, |
263 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, | 279 | NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, |
264 | false, false, "ce"); | 280 | false, false, "ce"); |
265 | if (!mm->ce.vm) | 281 | if (!mm->ce.vm) { |
266 | return -ENOMEM; | 282 | return -ENOMEM; |
283 | } | ||
267 | return 0; | 284 | return 0; |
268 | } | 285 | } |
269 | 286 | ||
@@ -286,24 +303,30 @@ void nvgpu_init_mm_ce_context(struct gk20a *g) | |||
286 | 303 | ||
287 | static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) | 304 | static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) |
288 | { | 305 | { |
289 | if (g->ops.fb.reset) | 306 | if (g->ops.fb.reset) { |
290 | g->ops.fb.reset(g); | 307 | g->ops.fb.reset(g); |
308 | } | ||
291 | 309 | ||
292 | if (g->ops.clock_gating.slcg_fb_load_gating_prod) | 310 | if (g->ops.clock_gating.slcg_fb_load_gating_prod) { |
293 | g->ops.clock_gating.slcg_fb_load_gating_prod(g, | 311 | g->ops.clock_gating.slcg_fb_load_gating_prod(g, |
294 | g->slcg_enabled); | 312 | g->slcg_enabled); |
295 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) | 313 | } |
314 | if (g->ops.clock_gating.slcg_ltc_load_gating_prod) { | ||
296 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, | 315 | g->ops.clock_gating.slcg_ltc_load_gating_prod(g, |
297 | g->slcg_enabled); | 316 | g->slcg_enabled); |
298 | if (g->ops.clock_gating.blcg_fb_load_gating_prod) | 317 | } |
318 | if (g->ops.clock_gating.blcg_fb_load_gating_prod) { | ||
299 | g->ops.clock_gating.blcg_fb_load_gating_prod(g, | 319 | g->ops.clock_gating.blcg_fb_load_gating_prod(g, |
300 | g->blcg_enabled); | 320 | g->blcg_enabled); |
301 | if (g->ops.clock_gating.blcg_ltc_load_gating_prod) | 321 | } |
322 | if (g->ops.clock_gating.blcg_ltc_load_gating_prod) { | ||
302 | g->ops.clock_gating.blcg_ltc_load_gating_prod(g, | 323 | g->ops.clock_gating.blcg_ltc_load_gating_prod(g, |
303 | g->blcg_enabled); | 324 | g->blcg_enabled); |
325 | } | ||
304 | 326 | ||
305 | if (g->ops.fb.init_fs_state) | 327 | if (g->ops.fb.init_fs_state) { |
306 | g->ops.fb.init_fs_state(g); | 328 | g->ops.fb.init_fs_state(g); |
329 | } | ||
307 | 330 | ||
308 | return 0; | 331 | return 0; |
309 | } | 332 | } |
@@ -324,12 +347,14 @@ static int nvgpu_init_bar1_vm(struct mm_gk20a *mm) | |||
324 | mm->bar1.aperture_size, | 347 | mm->bar1.aperture_size, |
325 | true, false, | 348 | true, false, |
326 | "bar1"); | 349 | "bar1"); |
327 | if (!mm->bar1.vm) | 350 | if (!mm->bar1.vm) { |
328 | return -ENOMEM; | 351 | return -ENOMEM; |
352 | } | ||
329 | 353 | ||
330 | err = g->ops.mm.alloc_inst_block(g, inst_block); | 354 | err = g->ops.mm.alloc_inst_block(g, inst_block); |
331 | if (err) | 355 | if (err) { |
332 | goto clean_up_vm; | 356 | goto clean_up_vm; |
357 | } | ||
333 | g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); | 358 | g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); |
334 | 359 | ||
335 | return 0; | 360 | return 0; |
@@ -366,8 +391,9 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g) | |||
366 | mm->vidmem.ce_ctx_id = (u32)~0; | 391 | mm->vidmem.ce_ctx_id = (u32)~0; |
367 | 392 | ||
368 | err = nvgpu_vidmem_init(mm); | 393 | err = nvgpu_vidmem_init(mm); |
369 | if (err) | 394 | if (err) { |
370 | return err; | 395 | return err; |
396 | } | ||
371 | 397 | ||
372 | /* | 398 | /* |
373 | * this requires fixed allocations in vidmem which must be | 399 | * this requires fixed allocations in vidmem which must be |
@@ -376,40 +402,48 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g) | |||
376 | if (g->ops.pmu.alloc_blob_space | 402 | if (g->ops.pmu.alloc_blob_space |
377 | && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { | 403 | && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { |
378 | err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); | 404 | err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); |
379 | if (err) | 405 | if (err) { |
380 | return err; | 406 | return err; |
407 | } | ||
381 | } | 408 | } |
382 | 409 | ||
383 | err = nvgpu_alloc_sysmem_flush(g); | 410 | err = nvgpu_alloc_sysmem_flush(g); |
384 | if (err) | 411 | if (err) { |
385 | return err; | 412 | return err; |
413 | } | ||
386 | 414 | ||
387 | err = nvgpu_init_bar1_vm(mm); | 415 | err = nvgpu_init_bar1_vm(mm); |
388 | if (err) | 416 | if (err) { |
389 | return err; | 417 | return err; |
418 | } | ||
390 | 419 | ||
391 | if (g->ops.mm.init_bar2_vm) { | 420 | if (g->ops.mm.init_bar2_vm) { |
392 | err = g->ops.mm.init_bar2_vm(g); | 421 | err = g->ops.mm.init_bar2_vm(g); |
393 | if (err) | 422 | if (err) { |
394 | return err; | 423 | return err; |
424 | } | ||
395 | } | 425 | } |
396 | err = nvgpu_init_system_vm(mm); | 426 | err = nvgpu_init_system_vm(mm); |
397 | if (err) | 427 | if (err) { |
398 | return err; | 428 | return err; |
429 | } | ||
399 | 430 | ||
400 | err = nvgpu_init_hwpm(mm); | 431 | err = nvgpu_init_hwpm(mm); |
401 | if (err) | 432 | if (err) { |
402 | return err; | 433 | return err; |
434 | } | ||
403 | 435 | ||
404 | if (g->has_cde) { | 436 | if (g->has_cde) { |
405 | err = nvgpu_init_cde_vm(mm); | 437 | err = nvgpu_init_cde_vm(mm); |
406 | if (err) | 438 | if (err) { |
407 | return err; | 439 | return err; |
440 | } | ||
408 | } | 441 | } |
409 | 442 | ||
410 | err = nvgpu_init_ce_vm(mm); | 443 | err = nvgpu_init_ce_vm(mm); |
411 | if (err) | 444 | if (err) { |
412 | return err; | 445 | return err; |
446 | } | ||
413 | 447 | ||
414 | mm->remove_support = nvgpu_remove_mm_support; | 448 | mm->remove_support = nvgpu_remove_mm_support; |
415 | mm->remove_ce_support = nvgpu_remove_mm_ce_support; | 449 | mm->remove_ce_support = nvgpu_remove_mm_ce_support; |
@@ -424,15 +458,18 @@ int nvgpu_init_mm_support(struct gk20a *g) | |||
424 | u32 err; | 458 | u32 err; |
425 | 459 | ||
426 | err = nvgpu_init_mm_reset_enable_hw(g); | 460 | err = nvgpu_init_mm_reset_enable_hw(g); |
427 | if (err) | 461 | if (err) { |
428 | return err; | 462 | return err; |
463 | } | ||
429 | 464 | ||
430 | err = nvgpu_init_mm_setup_sw(g); | 465 | err = nvgpu_init_mm_setup_sw(g); |
431 | if (err) | 466 | if (err) { |
432 | return err; | 467 | return err; |
468 | } | ||
433 | 469 | ||
434 | if (g->ops.mm.init_mm_setup_hw) | 470 | if (g->ops.mm.init_mm_setup_hw) { |
435 | err = g->ops.mm.init_mm_setup_hw(g); | 471 | err = g->ops.mm.init_mm_setup_hw(g); |
472 | } | ||
436 | 473 | ||
437 | return err; | 474 | return err; |
438 | } | 475 | } |
@@ -443,8 +480,9 @@ u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g) | |||
443 | 480 | ||
444 | big_page_size = g->ops.mm.get_default_big_page_size(); | 481 | big_page_size = g->ops.mm.get_default_big_page_size(); |
445 | 482 | ||
446 | if (g->mm.disable_bigpage) | 483 | if (g->mm.disable_bigpage) { |
447 | big_page_size = 0; | 484 | big_page_size = 0; |
485 | } | ||
448 | 486 | ||
449 | return big_page_size; | 487 | return big_page_size; |
450 | } | 488 | } |
@@ -456,8 +494,9 @@ u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g) | |||
456 | if (!g->mm.disable_bigpage) { | 494 | if (!g->mm.disable_bigpage) { |
457 | available_big_page_sizes = | 495 | available_big_page_sizes = |
458 | g->ops.mm.get_default_big_page_size(); | 496 | g->ops.mm.get_default_big_page_size(); |
459 | if (g->ops.mm.get_big_page_sizes) | 497 | if (g->ops.mm.get_big_page_sizes) { |
460 | available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); | 498 | available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); |
499 | } | ||
461 | } | 500 | } |
462 | 501 | ||
463 | return available_big_page_sizes; | 502 | return available_big_page_sizes; |