summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/mm/mm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/mm/mm.c')
-rw-r--r--drivers/gpu/nvgpu/common/mm/mm.c119
1 files changed, 79 insertions, 40 deletions
diff --git a/drivers/gpu/nvgpu/common/mm/mm.c b/drivers/gpu/nvgpu/common/mm/mm.c
index 2c3a1cd6..0608d66a 100644
--- a/drivers/gpu/nvgpu/common/mm/mm.c
+++ b/drivers/gpu/nvgpu/common/mm/mm.c
@@ -40,8 +40,9 @@ enum gmmu_pgsz_gk20a __get_pte_size_fixed_map(struct vm_gk20a *vm,
40 struct nvgpu_vm_area *vm_area; 40 struct nvgpu_vm_area *vm_area;
41 41
42 vm_area = nvgpu_vm_area_find(vm, base); 42 vm_area = nvgpu_vm_area_find(vm, base);
43 if (!vm_area) 43 if (!vm_area) {
44 return gmmu_page_size_small; 44 return gmmu_page_size_small;
45 }
45 46
46 return vm_area->pgsz_idx; 47 return vm_area->pgsz_idx;
47} 48}
@@ -53,14 +54,16 @@ static enum gmmu_pgsz_gk20a __get_pte_size_split_addr(struct vm_gk20a *vm,
53 u64 base, u64 size) 54 u64 base, u64 size)
54{ 55{
55 if (!base) { 56 if (!base) {
56 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) 57 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big]) {
57 return gmmu_page_size_big; 58 return gmmu_page_size_big;
59 }
58 return gmmu_page_size_small; 60 return gmmu_page_size_small;
59 } else { 61 } else {
60 if (base < __nv_gmmu_va_small_page_limit()) 62 if (base < __nv_gmmu_va_small_page_limit()) {
61 return gmmu_page_size_small; 63 return gmmu_page_size_small;
62 else 64 } else {
63 return gmmu_page_size_big; 65 return gmmu_page_size_big;
66 }
64 } 67 }
65} 68}
66 69
@@ -89,18 +92,22 @@ enum gmmu_pgsz_gk20a __get_pte_size(struct vm_gk20a *vm, u64 base, u64 size)
89{ 92{
90 struct gk20a *g = gk20a_from_vm(vm); 93 struct gk20a *g = gk20a_from_vm(vm);
91 94
92 if (!vm->big_pages) 95 if (!vm->big_pages) {
93 return gmmu_page_size_small; 96 return gmmu_page_size_small;
97 }
94 98
95 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) 99 if (!nvgpu_is_enabled(g, NVGPU_MM_UNIFY_ADDRESS_SPACES)) {
96 return __get_pte_size_split_addr(vm, base, size); 100 return __get_pte_size_split_addr(vm, base, size);
101 }
97 102
98 if (base) 103 if (base) {
99 return __get_pte_size_fixed_map(vm, base, size); 104 return __get_pte_size_fixed_map(vm, base, size);
105 }
100 106
101 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] && 107 if (size >= vm->gmmu_page_sizes[gmmu_page_size_big] &&
102 nvgpu_iommuable(g)) 108 nvgpu_iommuable(g)) {
103 return gmmu_page_size_big; 109 return gmmu_page_size_big;
110 }
104 return gmmu_page_size_small; 111 return gmmu_page_size_small;
105} 112}
106 113
@@ -137,8 +144,9 @@ u64 nvgpu_inst_block_addr(struct gk20a *g, struct nvgpu_mem *inst_block)
137 144
138void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block) 145void nvgpu_free_inst_block(struct gk20a *g, struct nvgpu_mem *inst_block)
139{ 146{
140 if (nvgpu_mem_is_valid(inst_block)) 147 if (nvgpu_mem_is_valid(inst_block)) {
141 nvgpu_dma_free(g, inst_block); 148 nvgpu_dma_free(g, inst_block);
149 }
142} 150}
143 151
144static int nvgpu_alloc_sysmem_flush(struct gk20a *g) 152static int nvgpu_alloc_sysmem_flush(struct gk20a *g)
@@ -150,8 +158,9 @@ static void nvgpu_remove_mm_ce_support(struct mm_gk20a *mm)
150{ 158{
151 struct gk20a *g = gk20a_from_mm(mm); 159 struct gk20a *g = gk20a_from_mm(mm);
152 160
153 if (mm->vidmem.ce_ctx_id != (u32)~0) 161 if (mm->vidmem.ce_ctx_id != (u32)~0) {
154 gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id); 162 gk20a_ce_delete_context_priv(g, mm->vidmem.ce_ctx_id);
163 }
155 164
156 mm->vidmem.ce_ctx_id = (u32)~0; 165 mm->vidmem.ce_ctx_id = (u32)~0;
157 166
@@ -162,11 +171,13 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
162{ 171{
163 struct gk20a *g = gk20a_from_mm(mm); 172 struct gk20a *g = gk20a_from_mm(mm);
164 173
165 if (g->ops.mm.fault_info_mem_destroy) 174 if (g->ops.mm.fault_info_mem_destroy) {
166 g->ops.mm.fault_info_mem_destroy(g); 175 g->ops.mm.fault_info_mem_destroy(g);
176 }
167 177
168 if (g->ops.mm.remove_bar2_vm) 178 if (g->ops.mm.remove_bar2_vm) {
169 g->ops.mm.remove_bar2_vm(g); 179 g->ops.mm.remove_bar2_vm(g);
180 }
170 181
171 nvgpu_free_inst_block(g, &mm->bar1.inst_block); 182 nvgpu_free_inst_block(g, &mm->bar1.inst_block);
172 nvgpu_vm_put(mm->bar1.vm); 183 nvgpu_vm_put(mm->bar1.vm);
@@ -175,8 +186,9 @@ static void nvgpu_remove_mm_support(struct mm_gk20a *mm)
175 nvgpu_free_inst_block(g, &mm->hwpm.inst_block); 186 nvgpu_free_inst_block(g, &mm->hwpm.inst_block);
176 nvgpu_vm_put(mm->pmu.vm); 187 nvgpu_vm_put(mm->pmu.vm);
177 188
178 if (g->has_cde) 189 if (g->has_cde) {
179 nvgpu_vm_put(mm->cde.vm); 190 nvgpu_vm_put(mm->cde.vm);
191 }
180 192
181 nvgpu_semaphore_sea_destroy(g); 193 nvgpu_semaphore_sea_destroy(g);
182 nvgpu_vidmem_destroy(g); 194 nvgpu_vidmem_destroy(g);
@@ -208,12 +220,14 @@ static int nvgpu_init_system_vm(struct mm_gk20a *mm)
208 true, 220 true,
209 false, 221 false,
210 "system"); 222 "system");
211 if (!mm->pmu.vm) 223 if (!mm->pmu.vm) {
212 return -ENOMEM; 224 return -ENOMEM;
225 }
213 226
214 err = g->ops.mm.alloc_inst_block(g, inst_block); 227 err = g->ops.mm.alloc_inst_block(g, inst_block);
215 if (err) 228 if (err) {
216 goto clean_up_vm; 229 goto clean_up_vm;
230 }
217 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size); 231 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, big_page_size);
218 232
219 return 0; 233 return 0;
@@ -230,8 +244,9 @@ static int nvgpu_init_hwpm(struct mm_gk20a *mm)
230 struct nvgpu_mem *inst_block = &mm->hwpm.inst_block; 244 struct nvgpu_mem *inst_block = &mm->hwpm.inst_block;
231 245
232 err = g->ops.mm.alloc_inst_block(g, inst_block); 246 err = g->ops.mm.alloc_inst_block(g, inst_block);
233 if (err) 247 if (err) {
234 return err; 248 return err;
249 }
235 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0); 250 g->ops.mm.init_inst_block(inst_block, mm->pmu.vm, 0);
236 251
237 return 0; 252 return 0;
@@ -247,8 +262,9 @@ static int nvgpu_init_cde_vm(struct mm_gk20a *mm)
247 NV_MM_DEFAULT_KERNEL_SIZE, 262 NV_MM_DEFAULT_KERNEL_SIZE,
248 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, 263 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
249 false, false, "cde"); 264 false, false, "cde");
250 if (!mm->cde.vm) 265 if (!mm->cde.vm) {
251 return -ENOMEM; 266 return -ENOMEM;
267 }
252 return 0; 268 return 0;
253} 269}
254 270
@@ -262,8 +278,9 @@ static int nvgpu_init_ce_vm(struct mm_gk20a *mm)
262 NV_MM_DEFAULT_KERNEL_SIZE, 278 NV_MM_DEFAULT_KERNEL_SIZE,
263 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE, 279 NV_MM_DEFAULT_KERNEL_SIZE + NV_MM_DEFAULT_USER_SIZE,
264 false, false, "ce"); 280 false, false, "ce");
265 if (!mm->ce.vm) 281 if (!mm->ce.vm) {
266 return -ENOMEM; 282 return -ENOMEM;
283 }
267 return 0; 284 return 0;
268} 285}
269 286
@@ -286,24 +303,30 @@ void nvgpu_init_mm_ce_context(struct gk20a *g)
286 303
287static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g) 304static int nvgpu_init_mm_reset_enable_hw(struct gk20a *g)
288{ 305{
289 if (g->ops.fb.reset) 306 if (g->ops.fb.reset) {
290 g->ops.fb.reset(g); 307 g->ops.fb.reset(g);
308 }
291 309
292 if (g->ops.clock_gating.slcg_fb_load_gating_prod) 310 if (g->ops.clock_gating.slcg_fb_load_gating_prod) {
293 g->ops.clock_gating.slcg_fb_load_gating_prod(g, 311 g->ops.clock_gating.slcg_fb_load_gating_prod(g,
294 g->slcg_enabled); 312 g->slcg_enabled);
295 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) 313 }
314 if (g->ops.clock_gating.slcg_ltc_load_gating_prod) {
296 g->ops.clock_gating.slcg_ltc_load_gating_prod(g, 315 g->ops.clock_gating.slcg_ltc_load_gating_prod(g,
297 g->slcg_enabled); 316 g->slcg_enabled);
298 if (g->ops.clock_gating.blcg_fb_load_gating_prod) 317 }
318 if (g->ops.clock_gating.blcg_fb_load_gating_prod) {
299 g->ops.clock_gating.blcg_fb_load_gating_prod(g, 319 g->ops.clock_gating.blcg_fb_load_gating_prod(g,
300 g->blcg_enabled); 320 g->blcg_enabled);
301 if (g->ops.clock_gating.blcg_ltc_load_gating_prod) 321 }
322 if (g->ops.clock_gating.blcg_ltc_load_gating_prod) {
302 g->ops.clock_gating.blcg_ltc_load_gating_prod(g, 323 g->ops.clock_gating.blcg_ltc_load_gating_prod(g,
303 g->blcg_enabled); 324 g->blcg_enabled);
325 }
304 326
305 if (g->ops.fb.init_fs_state) 327 if (g->ops.fb.init_fs_state) {
306 g->ops.fb.init_fs_state(g); 328 g->ops.fb.init_fs_state(g);
329 }
307 330
308 return 0; 331 return 0;
309} 332}
@@ -324,12 +347,14 @@ static int nvgpu_init_bar1_vm(struct mm_gk20a *mm)
324 mm->bar1.aperture_size, 347 mm->bar1.aperture_size,
325 true, false, 348 true, false,
326 "bar1"); 349 "bar1");
327 if (!mm->bar1.vm) 350 if (!mm->bar1.vm) {
328 return -ENOMEM; 351 return -ENOMEM;
352 }
329 353
330 err = g->ops.mm.alloc_inst_block(g, inst_block); 354 err = g->ops.mm.alloc_inst_block(g, inst_block);
331 if (err) 355 if (err) {
332 goto clean_up_vm; 356 goto clean_up_vm;
357 }
333 g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size); 358 g->ops.mm.init_inst_block(inst_block, mm->bar1.vm, big_page_size);
334 359
335 return 0; 360 return 0;
@@ -366,8 +391,9 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
366 mm->vidmem.ce_ctx_id = (u32)~0; 391 mm->vidmem.ce_ctx_id = (u32)~0;
367 392
368 err = nvgpu_vidmem_init(mm); 393 err = nvgpu_vidmem_init(mm);
369 if (err) 394 if (err) {
370 return err; 395 return err;
396 }
371 397
372 /* 398 /*
373 * this requires fixed allocations in vidmem which must be 399 * this requires fixed allocations in vidmem which must be
@@ -376,40 +402,48 @@ static int nvgpu_init_mm_setup_sw(struct gk20a *g)
376 if (g->ops.pmu.alloc_blob_space 402 if (g->ops.pmu.alloc_blob_space
377 && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) { 403 && !nvgpu_is_enabled(g, NVGPU_MM_UNIFIED_MEMORY)) {
378 err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob); 404 err = g->ops.pmu.alloc_blob_space(g, 0, &g->acr.ucode_blob);
379 if (err) 405 if (err) {
380 return err; 406 return err;
407 }
381 } 408 }
382 409
383 err = nvgpu_alloc_sysmem_flush(g); 410 err = nvgpu_alloc_sysmem_flush(g);
384 if (err) 411 if (err) {
385 return err; 412 return err;
413 }
386 414
387 err = nvgpu_init_bar1_vm(mm); 415 err = nvgpu_init_bar1_vm(mm);
388 if (err) 416 if (err) {
389 return err; 417 return err;
418 }
390 419
391 if (g->ops.mm.init_bar2_vm) { 420 if (g->ops.mm.init_bar2_vm) {
392 err = g->ops.mm.init_bar2_vm(g); 421 err = g->ops.mm.init_bar2_vm(g);
393 if (err) 422 if (err) {
394 return err; 423 return err;
424 }
395 } 425 }
396 err = nvgpu_init_system_vm(mm); 426 err = nvgpu_init_system_vm(mm);
397 if (err) 427 if (err) {
398 return err; 428 return err;
429 }
399 430
400 err = nvgpu_init_hwpm(mm); 431 err = nvgpu_init_hwpm(mm);
401 if (err) 432 if (err) {
402 return err; 433 return err;
434 }
403 435
404 if (g->has_cde) { 436 if (g->has_cde) {
405 err = nvgpu_init_cde_vm(mm); 437 err = nvgpu_init_cde_vm(mm);
406 if (err) 438 if (err) {
407 return err; 439 return err;
440 }
408 } 441 }
409 442
410 err = nvgpu_init_ce_vm(mm); 443 err = nvgpu_init_ce_vm(mm);
411 if (err) 444 if (err) {
412 return err; 445 return err;
446 }
413 447
414 mm->remove_support = nvgpu_remove_mm_support; 448 mm->remove_support = nvgpu_remove_mm_support;
415 mm->remove_ce_support = nvgpu_remove_mm_ce_support; 449 mm->remove_ce_support = nvgpu_remove_mm_ce_support;
@@ -424,15 +458,18 @@ int nvgpu_init_mm_support(struct gk20a *g)
424 u32 err; 458 u32 err;
425 459
426 err = nvgpu_init_mm_reset_enable_hw(g); 460 err = nvgpu_init_mm_reset_enable_hw(g);
427 if (err) 461 if (err) {
428 return err; 462 return err;
463 }
429 464
430 err = nvgpu_init_mm_setup_sw(g); 465 err = nvgpu_init_mm_setup_sw(g);
431 if (err) 466 if (err) {
432 return err; 467 return err;
468 }
433 469
434 if (g->ops.mm.init_mm_setup_hw) 470 if (g->ops.mm.init_mm_setup_hw) {
435 err = g->ops.mm.init_mm_setup_hw(g); 471 err = g->ops.mm.init_mm_setup_hw(g);
472 }
436 473
437 return err; 474 return err;
438} 475}
@@ -443,8 +480,9 @@ u32 nvgpu_mm_get_default_big_page_size(struct gk20a *g)
443 480
444 big_page_size = g->ops.mm.get_default_big_page_size(); 481 big_page_size = g->ops.mm.get_default_big_page_size();
445 482
446 if (g->mm.disable_bigpage) 483 if (g->mm.disable_bigpage) {
447 big_page_size = 0; 484 big_page_size = 0;
485 }
448 486
449 return big_page_size; 487 return big_page_size;
450} 488}
@@ -456,8 +494,9 @@ u32 nvgpu_mm_get_available_big_page_sizes(struct gk20a *g)
456 if (!g->mm.disable_bigpage) { 494 if (!g->mm.disable_bigpage) {
457 available_big_page_sizes = 495 available_big_page_sizes =
458 g->ops.mm.get_default_big_page_size(); 496 g->ops.mm.get_default_big_page_size();
459 if (g->ops.mm.get_big_page_sizes) 497 if (g->ops.mm.get_big_page_sizes) {
460 available_big_page_sizes |= g->ops.mm.get_big_page_sizes(); 498 available_big_page_sizes |= g->ops.mm.get_big_page_sizes();
499 }
461 } 500 }
462 501
463 return available_big_page_sizes; 502 return available_big_page_sizes;