diff options
Diffstat (limited to 'arch/x86/mm/fault.c')
-rw-r--r-- | arch/x86/mm/fault.c | 60 |
1 files changed, 19 insertions, 41 deletions
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index f75ea0748b9f..73bd8c95ac71 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c | |||
@@ -417,11 +417,11 @@ void vmalloc_sync_all(void) | |||
417 | */ | 417 | */ |
418 | static noinline int vmalloc_fault(unsigned long address) | 418 | static noinline int vmalloc_fault(unsigned long address) |
419 | { | 419 | { |
420 | pgd_t *pgd, *pgd_ref; | 420 | pgd_t *pgd, *pgd_k; |
421 | p4d_t *p4d, *p4d_ref; | 421 | p4d_t *p4d, *p4d_k; |
422 | pud_t *pud, *pud_ref; | 422 | pud_t *pud; |
423 | pmd_t *pmd, *pmd_ref; | 423 | pmd_t *pmd; |
424 | pte_t *pte, *pte_ref; | 424 | pte_t *pte; |
425 | 425 | ||
426 | /* Make sure we are in vmalloc area: */ | 426 | /* Make sure we are in vmalloc area: */ |
427 | if (!(address >= VMALLOC_START && address < VMALLOC_END)) | 427 | if (!(address >= VMALLOC_START && address < VMALLOC_END)) |
@@ -435,73 +435,51 @@ static noinline int vmalloc_fault(unsigned long address) | |||
435 | * case just flush: | 435 | * case just flush: |
436 | */ | 436 | */ |
437 | pgd = (pgd_t *)__va(read_cr3_pa()) + pgd_index(address); | 437 | pgd = (pgd_t *)__va(read_cr3_pa()) + pgd_index(address); |
438 | pgd_ref = pgd_offset_k(address); | 438 | pgd_k = pgd_offset_k(address); |
439 | if (pgd_none(*pgd_ref)) | 439 | if (pgd_none(*pgd_k)) |
440 | return -1; | 440 | return -1; |
441 | 441 | ||
442 | if (CONFIG_PGTABLE_LEVELS > 4) { | 442 | if (pgtable_l5_enabled) { |
443 | if (pgd_none(*pgd)) { | 443 | if (pgd_none(*pgd)) { |
444 | set_pgd(pgd, *pgd_ref); | 444 | set_pgd(pgd, *pgd_k); |
445 | arch_flush_lazy_mmu_mode(); | 445 | arch_flush_lazy_mmu_mode(); |
446 | } else { | 446 | } else { |
447 | BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref)); | 447 | BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_k)); |
448 | } | 448 | } |
449 | } | 449 | } |
450 | 450 | ||
451 | /* With 4-level paging, copying happens on the p4d level. */ | 451 | /* With 4-level paging, copying happens on the p4d level. */ |
452 | p4d = p4d_offset(pgd, address); | 452 | p4d = p4d_offset(pgd, address); |
453 | p4d_ref = p4d_offset(pgd_ref, address); | 453 | p4d_k = p4d_offset(pgd_k, address); |
454 | if (p4d_none(*p4d_ref)) | 454 | if (p4d_none(*p4d_k)) |
455 | return -1; | 455 | return -1; |
456 | 456 | ||
457 | if (p4d_none(*p4d) && CONFIG_PGTABLE_LEVELS == 4) { | 457 | if (p4d_none(*p4d) && !pgtable_l5_enabled) { |
458 | set_p4d(p4d, *p4d_ref); | 458 | set_p4d(p4d, *p4d_k); |
459 | arch_flush_lazy_mmu_mode(); | 459 | arch_flush_lazy_mmu_mode(); |
460 | } else { | 460 | } else { |
461 | BUG_ON(p4d_pfn(*p4d) != p4d_pfn(*p4d_ref)); | 461 | BUG_ON(p4d_pfn(*p4d) != p4d_pfn(*p4d_k)); |
462 | } | 462 | } |
463 | 463 | ||
464 | /* | ||
465 | * Below here mismatches are bugs because these lower tables | ||
466 | * are shared: | ||
467 | */ | ||
468 | BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS < 4); | 464 | BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS < 4); |
469 | 465 | ||
470 | pud = pud_offset(p4d, address); | 466 | pud = pud_offset(p4d, address); |
471 | pud_ref = pud_offset(p4d_ref, address); | 467 | if (pud_none(*pud)) |
472 | if (pud_none(*pud_ref)) | ||
473 | return -1; | 468 | return -1; |
474 | 469 | ||
475 | if (pud_none(*pud) || pud_pfn(*pud) != pud_pfn(*pud_ref)) | ||
476 | BUG(); | ||
477 | |||
478 | if (pud_large(*pud)) | 470 | if (pud_large(*pud)) |
479 | return 0; | 471 | return 0; |
480 | 472 | ||
481 | pmd = pmd_offset(pud, address); | 473 | pmd = pmd_offset(pud, address); |
482 | pmd_ref = pmd_offset(pud_ref, address); | 474 | if (pmd_none(*pmd)) |
483 | if (pmd_none(*pmd_ref)) | ||
484 | return -1; | 475 | return -1; |
485 | 476 | ||
486 | if (pmd_none(*pmd) || pmd_pfn(*pmd) != pmd_pfn(*pmd_ref)) | ||
487 | BUG(); | ||
488 | |||
489 | if (pmd_large(*pmd)) | 477 | if (pmd_large(*pmd)) |
490 | return 0; | 478 | return 0; |
491 | 479 | ||
492 | pte_ref = pte_offset_kernel(pmd_ref, address); | ||
493 | if (!pte_present(*pte_ref)) | ||
494 | return -1; | ||
495 | |||
496 | pte = pte_offset_kernel(pmd, address); | 480 | pte = pte_offset_kernel(pmd, address); |
497 | 481 | if (!pte_present(*pte)) | |
498 | /* | 482 | return -1; |
499 | * Don't use pte_page here, because the mappings can point | ||
500 | * outside mem_map, and the NUMA hash lookup cannot handle | ||
501 | * that: | ||
502 | */ | ||
503 | if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref)) | ||
504 | BUG(); | ||
505 | 483 | ||
506 | return 0; | 484 | return 0; |
507 | } | 485 | } |