diff options
Diffstat (limited to 'arch/x86/mm')
-rw-r--r-- | arch/x86/mm/fault.c | 11 | ||||
-rw-r--r-- | arch/x86/mm/init_32.c | 16 | ||||
-rw-r--r-- | arch/x86/mm/init_64.c | 2 | ||||
-rw-r--r-- | arch/x86/mm/pat.c | 236 |
4 files changed, 249 insertions, 16 deletions
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 31e8730fa246..20ef272c412c 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c | |||
@@ -413,6 +413,7 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs, | |||
413 | unsigned long error_code) | 413 | unsigned long error_code) |
414 | { | 414 | { |
415 | unsigned long flags = oops_begin(); | 415 | unsigned long flags = oops_begin(); |
416 | int sig = SIGKILL; | ||
416 | struct task_struct *tsk; | 417 | struct task_struct *tsk; |
417 | 418 | ||
418 | printk(KERN_ALERT "%s: Corrupted page table at address %lx\n", | 419 | printk(KERN_ALERT "%s: Corrupted page table at address %lx\n", |
@@ -423,8 +424,8 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs, | |||
423 | tsk->thread.trap_no = 14; | 424 | tsk->thread.trap_no = 14; |
424 | tsk->thread.error_code = error_code; | 425 | tsk->thread.error_code = error_code; |
425 | if (__die("Bad pagetable", regs, error_code)) | 426 | if (__die("Bad pagetable", regs, error_code)) |
426 | regs = NULL; | 427 | sig = 0; |
427 | oops_end(flags, regs, SIGKILL); | 428 | oops_end(flags, regs, sig); |
428 | } | 429 | } |
429 | #endif | 430 | #endif |
430 | 431 | ||
@@ -590,6 +591,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code) | |||
590 | int fault; | 591 | int fault; |
591 | #ifdef CONFIG_X86_64 | 592 | #ifdef CONFIG_X86_64 |
592 | unsigned long flags; | 593 | unsigned long flags; |
594 | int sig; | ||
593 | #endif | 595 | #endif |
594 | 596 | ||
595 | tsk = current; | 597 | tsk = current; |
@@ -849,11 +851,12 @@ no_context: | |||
849 | bust_spinlocks(0); | 851 | bust_spinlocks(0); |
850 | do_exit(SIGKILL); | 852 | do_exit(SIGKILL); |
851 | #else | 853 | #else |
854 | sig = SIGKILL; | ||
852 | if (__die("Oops", regs, error_code)) | 855 | if (__die("Oops", regs, error_code)) |
853 | regs = NULL; | 856 | sig = 0; |
854 | /* Executive summary in case the body of the oops scrolled away */ | 857 | /* Executive summary in case the body of the oops scrolled away */ |
855 | printk(KERN_EMERG "CR2: %016lx\n", address); | 858 | printk(KERN_EMERG "CR2: %016lx\n", address); |
856 | oops_end(flags, regs, SIGKILL); | 859 | oops_end(flags, regs, sig); |
857 | #endif | 860 | #endif |
858 | 861 | ||
859 | /* | 862 | /* |
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index d3a45d54547a..800e1d94c1b5 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c | |||
@@ -67,7 +67,7 @@ static unsigned long __meminitdata table_top; | |||
67 | 67 | ||
68 | static int __initdata after_init_bootmem; | 68 | static int __initdata after_init_bootmem; |
69 | 69 | ||
70 | static __init void *alloc_low_page(unsigned long *phys) | 70 | static __init void *alloc_low_page(void) |
71 | { | 71 | { |
72 | unsigned long pfn = table_end++; | 72 | unsigned long pfn = table_end++; |
73 | void *adr; | 73 | void *adr; |
@@ -77,7 +77,6 @@ static __init void *alloc_low_page(unsigned long *phys) | |||
77 | 77 | ||
78 | adr = __va(pfn * PAGE_SIZE); | 78 | adr = __va(pfn * PAGE_SIZE); |
79 | memset(adr, 0, PAGE_SIZE); | 79 | memset(adr, 0, PAGE_SIZE); |
80 | *phys = pfn * PAGE_SIZE; | ||
81 | return adr; | 80 | return adr; |
82 | } | 81 | } |
83 | 82 | ||
@@ -92,16 +91,17 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd) | |||
92 | pmd_t *pmd_table; | 91 | pmd_t *pmd_table; |
93 | 92 | ||
94 | #ifdef CONFIG_X86_PAE | 93 | #ifdef CONFIG_X86_PAE |
95 | unsigned long phys; | ||
96 | if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { | 94 | if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { |
97 | if (after_init_bootmem) | 95 | if (after_init_bootmem) |
98 | pmd_table = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE); | 96 | pmd_table = (pmd_t *)alloc_bootmem_low_pages(PAGE_SIZE); |
99 | else | 97 | else |
100 | pmd_table = (pmd_t *)alloc_low_page(&phys); | 98 | pmd_table = (pmd_t *)alloc_low_page(); |
101 | paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT); | 99 | paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT); |
102 | set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); | 100 | set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); |
103 | pud = pud_offset(pgd, 0); | 101 | pud = pud_offset(pgd, 0); |
104 | BUG_ON(pmd_table != pmd_offset(pud, 0)); | 102 | BUG_ON(pmd_table != pmd_offset(pud, 0)); |
103 | |||
104 | return pmd_table; | ||
105 | } | 105 | } |
106 | #endif | 106 | #endif |
107 | pud = pud_offset(pgd, 0); | 107 | pud = pud_offset(pgd, 0); |
@@ -126,10 +126,8 @@ static pte_t * __init one_page_table_init(pmd_t *pmd) | |||
126 | if (!page_table) | 126 | if (!page_table) |
127 | page_table = | 127 | page_table = |
128 | (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); | 128 | (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); |
129 | } else { | 129 | } else |
130 | unsigned long phys; | 130 | page_table = (pte_t *)alloc_low_page(); |
131 | page_table = (pte_t *)alloc_low_page(&phys); | ||
132 | } | ||
133 | 131 | ||
134 | paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT); | 132 | paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT); |
135 | set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); | 133 | set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE)); |
@@ -969,8 +967,6 @@ void __init mem_init(void) | |||
969 | int codesize, reservedpages, datasize, initsize; | 967 | int codesize, reservedpages, datasize, initsize; |
970 | int tmp; | 968 | int tmp; |
971 | 969 | ||
972 | start_periodic_check_for_corruption(); | ||
973 | |||
974 | #ifdef CONFIG_FLATMEM | 970 | #ifdef CONFIG_FLATMEM |
975 | BUG_ON(!mem_map); | 971 | BUG_ON(!mem_map); |
976 | #endif | 972 | #endif |
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 9db01db6e3cd..9f7a0d24d42a 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -902,8 +902,6 @@ void __init mem_init(void) | |||
902 | long codesize, reservedpages, datasize, initsize; | 902 | long codesize, reservedpages, datasize, initsize; |
903 | unsigned long absent_pages; | 903 | unsigned long absent_pages; |
904 | 904 | ||
905 | start_periodic_check_for_corruption(); | ||
906 | |||
907 | pci_iommu_alloc(); | 905 | pci_iommu_alloc(); |
908 | 906 | ||
909 | /* clear_bss() already clear the empty_zero_page */ | 907 | /* clear_bss() already clear the empty_zero_page */ |
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c index eb1bf000d12e..541bcc944a5b 100644 --- a/arch/x86/mm/pat.c +++ b/arch/x86/mm/pat.c | |||
@@ -596,6 +596,242 @@ void unmap_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot) | |||
596 | free_memtype(addr, addr + size); | 596 | free_memtype(addr, addr + size); |
597 | } | 597 | } |
598 | 598 | ||
599 | /* | ||
600 | * Internal interface to reserve a range of physical memory with prot. | ||
601 | * Reserved non RAM regions only and after successful reserve_memtype, | ||
602 | * this func also keeps identity mapping (if any) in sync with this new prot. | ||
603 | */ | ||
604 | static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t vma_prot) | ||
605 | { | ||
606 | int is_ram = 0; | ||
607 | int id_sz, ret; | ||
608 | unsigned long flags; | ||
609 | unsigned long want_flags = (pgprot_val(vma_prot) & _PAGE_CACHE_MASK); | ||
610 | |||
611 | is_ram = pagerange_is_ram(paddr, paddr + size); | ||
612 | |||
613 | if (is_ram != 0) { | ||
614 | /* | ||
615 | * For mapping RAM pages, drivers need to call | ||
616 | * set_memory_[uc|wc|wb] directly, for reserve and free, before | ||
617 | * setting up the PTE. | ||
618 | */ | ||
619 | WARN_ON_ONCE(1); | ||
620 | return 0; | ||
621 | } | ||
622 | |||
623 | ret = reserve_memtype(paddr, paddr + size, want_flags, &flags); | ||
624 | if (ret) | ||
625 | return ret; | ||
626 | |||
627 | if (flags != want_flags) { | ||
628 | free_memtype(paddr, paddr + size); | ||
629 | printk(KERN_ERR | ||
630 | "%s:%d map pfn expected mapping type %s for %Lx-%Lx, got %s\n", | ||
631 | current->comm, current->pid, | ||
632 | cattr_name(want_flags), | ||
633 | (unsigned long long)paddr, | ||
634 | (unsigned long long)(paddr + size), | ||
635 | cattr_name(flags)); | ||
636 | return -EINVAL; | ||
637 | } | ||
638 | |||
639 | /* Need to keep identity mapping in sync */ | ||
640 | if (paddr >= __pa(high_memory)) | ||
641 | return 0; | ||
642 | |||
643 | id_sz = (__pa(high_memory) < paddr + size) ? | ||
644 | __pa(high_memory) - paddr : | ||
645 | size; | ||
646 | |||
647 | if (ioremap_change_attr((unsigned long)__va(paddr), id_sz, flags) < 0) { | ||
648 | free_memtype(paddr, paddr + size); | ||
649 | printk(KERN_ERR | ||
650 | "%s:%d reserve_pfn_range ioremap_change_attr failed %s " | ||
651 | "for %Lx-%Lx\n", | ||
652 | current->comm, current->pid, | ||
653 | cattr_name(flags), | ||
654 | (unsigned long long)paddr, | ||
655 | (unsigned long long)(paddr + size)); | ||
656 | return -EINVAL; | ||
657 | } | ||
658 | return 0; | ||
659 | } | ||
660 | |||
661 | /* | ||
662 | * Internal interface to free a range of physical memory. | ||
663 | * Frees non RAM regions only. | ||
664 | */ | ||
665 | static void free_pfn_range(u64 paddr, unsigned long size) | ||
666 | { | ||
667 | int is_ram; | ||
668 | |||
669 | is_ram = pagerange_is_ram(paddr, paddr + size); | ||
670 | if (is_ram == 0) | ||
671 | free_memtype(paddr, paddr + size); | ||
672 | } | ||
673 | |||
674 | /* | ||
675 | * track_pfn_vma_copy is called when vma that is covering the pfnmap gets | ||
676 | * copied through copy_page_range(). | ||
677 | * | ||
678 | * If the vma has a linear pfn mapping for the entire range, we get the prot | ||
679 | * from pte and reserve the entire vma range with single reserve_pfn_range call. | ||
680 | * Otherwise, we reserve the entire vma range, my ging through the PTEs page | ||
681 | * by page to get physical address and protection. | ||
682 | */ | ||
683 | int track_pfn_vma_copy(struct vm_area_struct *vma) | ||
684 | { | ||
685 | int retval = 0; | ||
686 | unsigned long i, j; | ||
687 | u64 paddr; | ||
688 | unsigned long prot; | ||
689 | unsigned long vma_start = vma->vm_start; | ||
690 | unsigned long vma_end = vma->vm_end; | ||
691 | unsigned long vma_size = vma_end - vma_start; | ||
692 | |||
693 | if (!pat_enabled) | ||
694 | return 0; | ||
695 | |||
696 | if (is_linear_pfn_mapping(vma)) { | ||
697 | /* | ||
698 | * reserve the whole chunk covered by vma. We need the | ||
699 | * starting address and protection from pte. | ||
700 | */ | ||
701 | if (follow_phys(vma, vma_start, 0, &prot, &paddr)) { | ||
702 | WARN_ON_ONCE(1); | ||
703 | return -EINVAL; | ||
704 | } | ||
705 | return reserve_pfn_range(paddr, vma_size, __pgprot(prot)); | ||
706 | } | ||
707 | |||
708 | /* reserve entire vma page by page, using pfn and prot from pte */ | ||
709 | for (i = 0; i < vma_size; i += PAGE_SIZE) { | ||
710 | if (follow_phys(vma, vma_start + i, 0, &prot, &paddr)) | ||
711 | continue; | ||
712 | |||
713 | retval = reserve_pfn_range(paddr, PAGE_SIZE, __pgprot(prot)); | ||
714 | if (retval) | ||
715 | goto cleanup_ret; | ||
716 | } | ||
717 | return 0; | ||
718 | |||
719 | cleanup_ret: | ||
720 | /* Reserve error: Cleanup partial reservation and return error */ | ||
721 | for (j = 0; j < i; j += PAGE_SIZE) { | ||
722 | if (follow_phys(vma, vma_start + j, 0, &prot, &paddr)) | ||
723 | continue; | ||
724 | |||
725 | free_pfn_range(paddr, PAGE_SIZE); | ||
726 | } | ||
727 | |||
728 | return retval; | ||
729 | } | ||
730 | |||
731 | /* | ||
732 | * track_pfn_vma_new is called when a _new_ pfn mapping is being established | ||
733 | * for physical range indicated by pfn and size. | ||
734 | * | ||
735 | * prot is passed in as a parameter for the new mapping. If the vma has a | ||
736 | * linear pfn mapping for the entire range reserve the entire vma range with | ||
737 | * single reserve_pfn_range call. | ||
738 | * Otherwise, we look t the pfn and size and reserve only the specified range | ||
739 | * page by page. | ||
740 | * | ||
741 | * Note that this function can be called with caller trying to map only a | ||
742 | * subrange/page inside the vma. | ||
743 | */ | ||
744 | int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t prot, | ||
745 | unsigned long pfn, unsigned long size) | ||
746 | { | ||
747 | int retval = 0; | ||
748 | unsigned long i, j; | ||
749 | u64 base_paddr; | ||
750 | u64 paddr; | ||
751 | unsigned long vma_start = vma->vm_start; | ||
752 | unsigned long vma_end = vma->vm_end; | ||
753 | unsigned long vma_size = vma_end - vma_start; | ||
754 | |||
755 | if (!pat_enabled) | ||
756 | return 0; | ||
757 | |||
758 | if (is_linear_pfn_mapping(vma)) { | ||
759 | /* reserve the whole chunk starting from vm_pgoff */ | ||
760 | paddr = (u64)vma->vm_pgoff << PAGE_SHIFT; | ||
761 | return reserve_pfn_range(paddr, vma_size, prot); | ||
762 | } | ||
763 | |||
764 | /* reserve page by page using pfn and size */ | ||
765 | base_paddr = (u64)pfn << PAGE_SHIFT; | ||
766 | for (i = 0; i < size; i += PAGE_SIZE) { | ||
767 | paddr = base_paddr + i; | ||
768 | retval = reserve_pfn_range(paddr, PAGE_SIZE, prot); | ||
769 | if (retval) | ||
770 | goto cleanup_ret; | ||
771 | } | ||
772 | return 0; | ||
773 | |||
774 | cleanup_ret: | ||
775 | /* Reserve error: Cleanup partial reservation and return error */ | ||
776 | for (j = 0; j < i; j += PAGE_SIZE) { | ||
777 | paddr = base_paddr + j; | ||
778 | free_pfn_range(paddr, PAGE_SIZE); | ||
779 | } | ||
780 | |||
781 | return retval; | ||
782 | } | ||
783 | |||
784 | /* | ||
785 | * untrack_pfn_vma is called while unmapping a pfnmap for a region. | ||
786 | * untrack can be called for a specific region indicated by pfn and size or | ||
787 | * can be for the entire vma (in which case size can be zero). | ||
788 | */ | ||
789 | void untrack_pfn_vma(struct vm_area_struct *vma, unsigned long pfn, | ||
790 | unsigned long size) | ||
791 | { | ||
792 | unsigned long i; | ||
793 | u64 paddr; | ||
794 | unsigned long prot; | ||
795 | unsigned long vma_start = vma->vm_start; | ||
796 | unsigned long vma_end = vma->vm_end; | ||
797 | unsigned long vma_size = vma_end - vma_start; | ||
798 | |||
799 | if (!pat_enabled) | ||
800 | return; | ||
801 | |||
802 | if (is_linear_pfn_mapping(vma)) { | ||
803 | /* free the whole chunk starting from vm_pgoff */ | ||
804 | paddr = (u64)vma->vm_pgoff << PAGE_SHIFT; | ||
805 | free_pfn_range(paddr, vma_size); | ||
806 | return; | ||
807 | } | ||
808 | |||
809 | if (size != 0 && size != vma_size) { | ||
810 | /* free page by page, using pfn and size */ | ||
811 | paddr = (u64)pfn << PAGE_SHIFT; | ||
812 | for (i = 0; i < size; i += PAGE_SIZE) { | ||
813 | paddr = paddr + i; | ||
814 | free_pfn_range(paddr, PAGE_SIZE); | ||
815 | } | ||
816 | } else { | ||
817 | /* free entire vma, page by page, using the pfn from pte */ | ||
818 | for (i = 0; i < vma_size; i += PAGE_SIZE) { | ||
819 | if (follow_phys(vma, vma_start + i, 0, &prot, &paddr)) | ||
820 | continue; | ||
821 | |||
822 | free_pfn_range(paddr, PAGE_SIZE); | ||
823 | } | ||
824 | } | ||
825 | } | ||
826 | |||
827 | pgprot_t pgprot_writecombine(pgprot_t prot) | ||
828 | { | ||
829 | if (pat_enabled) | ||
830 | return __pgprot(pgprot_val(prot) | _PAGE_CACHE_WC); | ||
831 | else | ||
832 | return pgprot_noncached(prot); | ||
833 | } | ||
834 | |||
599 | #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT) | 835 | #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT) |
600 | 836 | ||
601 | /* get Nth element of the linked list */ | 837 | /* get Nth element of the linked list */ |