diff options
Diffstat (limited to 'arch/x86/mm/pat.c')
-rw-r--r-- | arch/x86/mm/pat.c | 189 |
1 files changed, 32 insertions, 157 deletions
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c index c009a241d562..e6718bb28065 100644 --- a/arch/x86/mm/pat.c +++ b/arch/x86/mm/pat.c | |||
@@ -182,10 +182,10 @@ static unsigned long pat_x_mtrr_type(u64 start, u64 end, unsigned long req_type) | |||
182 | u8 mtrr_type; | 182 | u8 mtrr_type; |
183 | 183 | ||
184 | mtrr_type = mtrr_type_lookup(start, end); | 184 | mtrr_type = mtrr_type_lookup(start, end); |
185 | if (mtrr_type == MTRR_TYPE_UNCACHABLE) | 185 | if (mtrr_type != MTRR_TYPE_WRBACK) |
186 | return _PAGE_CACHE_UC; | 186 | return _PAGE_CACHE_UC_MINUS; |
187 | if (mtrr_type == MTRR_TYPE_WRCOMB) | 187 | |
188 | return _PAGE_CACHE_WC; | 188 | return _PAGE_CACHE_WB; |
189 | } | 189 | } |
190 | 190 | ||
191 | return req_type; | 191 | return req_type; |
@@ -352,23 +352,13 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type, | |||
352 | return 0; | 352 | return 0; |
353 | } | 353 | } |
354 | 354 | ||
355 | if (req_type == -1) { | 355 | /* |
356 | /* | 356 | * Call mtrr_lookup to get the type hint. This is an |
357 | * Call mtrr_lookup to get the type hint. This is an | 357 | * optimization for /dev/mem mmap'ers into WB memory (BIOS |
358 | * optimization for /dev/mem mmap'ers into WB memory (BIOS | 358 | * tools and ACPI tools). Use WB request for WB memory and use |
359 | * tools and ACPI tools). Use WB request for WB memory and use | 359 | * UC_MINUS otherwise. |
360 | * UC_MINUS otherwise. | 360 | */ |
361 | */ | 361 | actual_type = pat_x_mtrr_type(start, end, req_type & _PAGE_CACHE_MASK); |
362 | u8 mtrr_type = mtrr_type_lookup(start, end); | ||
363 | |||
364 | if (mtrr_type == MTRR_TYPE_WRBACK) | ||
365 | actual_type = _PAGE_CACHE_WB; | ||
366 | else | ||
367 | actual_type = _PAGE_CACHE_UC_MINUS; | ||
368 | } else { | ||
369 | actual_type = pat_x_mtrr_type(start, end, | ||
370 | req_type & _PAGE_CACHE_MASK); | ||
371 | } | ||
372 | 362 | ||
373 | if (new_type) | 363 | if (new_type) |
374 | *new_type = actual_type; | 364 | *new_type = actual_type; |
@@ -546,9 +536,7 @@ static inline int range_is_allowed(unsigned long pfn, unsigned long size) | |||
546 | int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, | 536 | int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, |
547 | unsigned long size, pgprot_t *vma_prot) | 537 | unsigned long size, pgprot_t *vma_prot) |
548 | { | 538 | { |
549 | u64 offset = ((u64) pfn) << PAGE_SHIFT; | 539 | unsigned long flags = _PAGE_CACHE_WB; |
550 | unsigned long flags = -1; | ||
551 | int retval; | ||
552 | 540 | ||
553 | if (!range_is_allowed(pfn, size)) | 541 | if (!range_is_allowed(pfn, size)) |
554 | return 0; | 542 | return 0; |
@@ -576,64 +564,11 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, | |||
576 | } | 564 | } |
577 | #endif | 565 | #endif |
578 | 566 | ||
579 | /* | ||
580 | * With O_SYNC, we can only take UC_MINUS mapping. Fail if we cannot. | ||
581 | * | ||
582 | * Without O_SYNC, we want to get | ||
583 | * - WB for WB-able memory and no other conflicting mappings | ||
584 | * - UC_MINUS for non-WB-able memory with no other conflicting mappings | ||
585 | * - Inherit from confliting mappings otherwise | ||
586 | */ | ||
587 | if (flags != -1) { | ||
588 | retval = reserve_memtype(offset, offset + size, flags, NULL); | ||
589 | } else { | ||
590 | retval = reserve_memtype(offset, offset + size, -1, &flags); | ||
591 | } | ||
592 | |||
593 | if (retval < 0) | ||
594 | return 0; | ||
595 | |||
596 | if (((pfn < max_low_pfn_mapped) || | ||
597 | (pfn >= (1UL<<(32 - PAGE_SHIFT)) && pfn < max_pfn_mapped)) && | ||
598 | ioremap_change_attr((unsigned long)__va(offset), size, flags) < 0) { | ||
599 | free_memtype(offset, offset + size); | ||
600 | printk(KERN_INFO | ||
601 | "%s:%d /dev/mem ioremap_change_attr failed %s for %Lx-%Lx\n", | ||
602 | current->comm, current->pid, | ||
603 | cattr_name(flags), | ||
604 | offset, (unsigned long long)(offset + size)); | ||
605 | return 0; | ||
606 | } | ||
607 | |||
608 | *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) | | 567 | *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) | |
609 | flags); | 568 | flags); |
610 | return 1; | 569 | return 1; |
611 | } | 570 | } |
612 | 571 | ||
613 | void map_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot) | ||
614 | { | ||
615 | unsigned long want_flags = (pgprot_val(vma_prot) & _PAGE_CACHE_MASK); | ||
616 | u64 addr = (u64)pfn << PAGE_SHIFT; | ||
617 | unsigned long flags; | ||
618 | |||
619 | reserve_memtype(addr, addr + size, want_flags, &flags); | ||
620 | if (flags != want_flags) { | ||
621 | printk(KERN_INFO | ||
622 | "%s:%d /dev/mem expected mapping type %s for %Lx-%Lx, got %s\n", | ||
623 | current->comm, current->pid, | ||
624 | cattr_name(want_flags), | ||
625 | addr, (unsigned long long)(addr + size), | ||
626 | cattr_name(flags)); | ||
627 | } | ||
628 | } | ||
629 | |||
630 | void unmap_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot) | ||
631 | { | ||
632 | u64 addr = (u64)pfn << PAGE_SHIFT; | ||
633 | |||
634 | free_memtype(addr, addr + size); | ||
635 | } | ||
636 | |||
637 | /* | 572 | /* |
638 | * Change the memory type for the physial address range in kernel identity | 573 | * Change the memory type for the physial address range in kernel identity |
639 | * mapping space if that range is a part of identity map. | 574 | * mapping space if that range is a part of identity map. |
@@ -671,8 +606,8 @@ static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot, | |||
671 | { | 606 | { |
672 | int is_ram = 0; | 607 | int is_ram = 0; |
673 | int ret; | 608 | int ret; |
674 | unsigned long flags; | ||
675 | unsigned long want_flags = (pgprot_val(*vma_prot) & _PAGE_CACHE_MASK); | 609 | unsigned long want_flags = (pgprot_val(*vma_prot) & _PAGE_CACHE_MASK); |
610 | unsigned long flags = want_flags; | ||
676 | 611 | ||
677 | is_ram = pat_pagerange_is_ram(paddr, paddr + size); | 612 | is_ram = pat_pagerange_is_ram(paddr, paddr + size); |
678 | 613 | ||
@@ -734,29 +669,28 @@ static void free_pfn_range(u64 paddr, unsigned long size) | |||
734 | * | 669 | * |
735 | * If the vma has a linear pfn mapping for the entire range, we get the prot | 670 | * If the vma has a linear pfn mapping for the entire range, we get the prot |
736 | * from pte and reserve the entire vma range with single reserve_pfn_range call. | 671 | * from pte and reserve the entire vma range with single reserve_pfn_range call. |
737 | * Otherwise, we reserve the entire vma range, my ging through the PTEs page | ||
738 | * by page to get physical address and protection. | ||
739 | */ | 672 | */ |
740 | int track_pfn_vma_copy(struct vm_area_struct *vma) | 673 | int track_pfn_vma_copy(struct vm_area_struct *vma) |
741 | { | 674 | { |
742 | int retval = 0; | ||
743 | unsigned long i, j; | ||
744 | resource_size_t paddr; | 675 | resource_size_t paddr; |
745 | unsigned long prot; | 676 | unsigned long prot; |
746 | unsigned long vma_start = vma->vm_start; | 677 | unsigned long vma_size = vma->vm_end - vma->vm_start; |
747 | unsigned long vma_end = vma->vm_end; | ||
748 | unsigned long vma_size = vma_end - vma_start; | ||
749 | pgprot_t pgprot; | 678 | pgprot_t pgprot; |
750 | 679 | ||
751 | if (!pat_enabled) | 680 | if (!pat_enabled) |
752 | return 0; | 681 | return 0; |
753 | 682 | ||
683 | /* | ||
684 | * For now, only handle remap_pfn_range() vmas where | ||
685 | * is_linear_pfn_mapping() == TRUE. Handling of | ||
686 | * vm_insert_pfn() is TBD. | ||
687 | */ | ||
754 | if (is_linear_pfn_mapping(vma)) { | 688 | if (is_linear_pfn_mapping(vma)) { |
755 | /* | 689 | /* |
756 | * reserve the whole chunk covered by vma. We need the | 690 | * reserve the whole chunk covered by vma. We need the |
757 | * starting address and protection from pte. | 691 | * starting address and protection from pte. |
758 | */ | 692 | */ |
759 | if (follow_phys(vma, vma_start, 0, &prot, &paddr)) { | 693 | if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) { |
760 | WARN_ON_ONCE(1); | 694 | WARN_ON_ONCE(1); |
761 | return -EINVAL; | 695 | return -EINVAL; |
762 | } | 696 | } |
@@ -764,28 +698,7 @@ int track_pfn_vma_copy(struct vm_area_struct *vma) | |||
764 | return reserve_pfn_range(paddr, vma_size, &pgprot, 1); | 698 | return reserve_pfn_range(paddr, vma_size, &pgprot, 1); |
765 | } | 699 | } |
766 | 700 | ||
767 | /* reserve entire vma page by page, using pfn and prot from pte */ | ||
768 | for (i = 0; i < vma_size; i += PAGE_SIZE) { | ||
769 | if (follow_phys(vma, vma_start + i, 0, &prot, &paddr)) | ||
770 | continue; | ||
771 | |||
772 | pgprot = __pgprot(prot); | ||
773 | retval = reserve_pfn_range(paddr, PAGE_SIZE, &pgprot, 1); | ||
774 | if (retval) | ||
775 | goto cleanup_ret; | ||
776 | } | ||
777 | return 0; | 701 | return 0; |
778 | |||
779 | cleanup_ret: | ||
780 | /* Reserve error: Cleanup partial reservation and return error */ | ||
781 | for (j = 0; j < i; j += PAGE_SIZE) { | ||
782 | if (follow_phys(vma, vma_start + j, 0, &prot, &paddr)) | ||
783 | continue; | ||
784 | |||
785 | free_pfn_range(paddr, PAGE_SIZE); | ||
786 | } | ||
787 | |||
788 | return retval; | ||
789 | } | 702 | } |
790 | 703 | ||
791 | /* | 704 | /* |
@@ -795,50 +708,28 @@ cleanup_ret: | |||
795 | * prot is passed in as a parameter for the new mapping. If the vma has a | 708 | * prot is passed in as a parameter for the new mapping. If the vma has a |
796 | * linear pfn mapping for the entire range reserve the entire vma range with | 709 | * linear pfn mapping for the entire range reserve the entire vma range with |
797 | * single reserve_pfn_range call. | 710 | * single reserve_pfn_range call. |
798 | * Otherwise, we look t the pfn and size and reserve only the specified range | ||
799 | * page by page. | ||
800 | * | ||
801 | * Note that this function can be called with caller trying to map only a | ||
802 | * subrange/page inside the vma. | ||
803 | */ | 711 | */ |
804 | int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t *prot, | 712 | int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t *prot, |
805 | unsigned long pfn, unsigned long size) | 713 | unsigned long pfn, unsigned long size) |
806 | { | 714 | { |
807 | int retval = 0; | ||
808 | unsigned long i, j; | ||
809 | resource_size_t base_paddr; | ||
810 | resource_size_t paddr; | 715 | resource_size_t paddr; |
811 | unsigned long vma_start = vma->vm_start; | 716 | unsigned long vma_size = vma->vm_end - vma->vm_start; |
812 | unsigned long vma_end = vma->vm_end; | ||
813 | unsigned long vma_size = vma_end - vma_start; | ||
814 | 717 | ||
815 | if (!pat_enabled) | 718 | if (!pat_enabled) |
816 | return 0; | 719 | return 0; |
817 | 720 | ||
721 | /* | ||
722 | * For now, only handle remap_pfn_range() vmas where | ||
723 | * is_linear_pfn_mapping() == TRUE. Handling of | ||
724 | * vm_insert_pfn() is TBD. | ||
725 | */ | ||
818 | if (is_linear_pfn_mapping(vma)) { | 726 | if (is_linear_pfn_mapping(vma)) { |
819 | /* reserve the whole chunk starting from vm_pgoff */ | 727 | /* reserve the whole chunk starting from vm_pgoff */ |
820 | paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT; | 728 | paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT; |
821 | return reserve_pfn_range(paddr, vma_size, prot, 0); | 729 | return reserve_pfn_range(paddr, vma_size, prot, 0); |
822 | } | 730 | } |
823 | 731 | ||
824 | /* reserve page by page using pfn and size */ | ||
825 | base_paddr = (resource_size_t)pfn << PAGE_SHIFT; | ||
826 | for (i = 0; i < size; i += PAGE_SIZE) { | ||
827 | paddr = base_paddr + i; | ||
828 | retval = reserve_pfn_range(paddr, PAGE_SIZE, prot, 0); | ||
829 | if (retval) | ||
830 | goto cleanup_ret; | ||
831 | } | ||
832 | return 0; | 732 | return 0; |
833 | |||
834 | cleanup_ret: | ||
835 | /* Reserve error: Cleanup partial reservation and return error */ | ||
836 | for (j = 0; j < i; j += PAGE_SIZE) { | ||
837 | paddr = base_paddr + j; | ||
838 | free_pfn_range(paddr, PAGE_SIZE); | ||
839 | } | ||
840 | |||
841 | return retval; | ||
842 | } | 733 | } |
843 | 734 | ||
844 | /* | 735 | /* |
@@ -849,39 +740,23 @@ cleanup_ret: | |||
849 | void untrack_pfn_vma(struct vm_area_struct *vma, unsigned long pfn, | 740 | void untrack_pfn_vma(struct vm_area_struct *vma, unsigned long pfn, |
850 | unsigned long size) | 741 | unsigned long size) |
851 | { | 742 | { |
852 | unsigned long i; | ||
853 | resource_size_t paddr; | 743 | resource_size_t paddr; |
854 | unsigned long prot; | 744 | unsigned long vma_size = vma->vm_end - vma->vm_start; |
855 | unsigned long vma_start = vma->vm_start; | ||
856 | unsigned long vma_end = vma->vm_end; | ||
857 | unsigned long vma_size = vma_end - vma_start; | ||
858 | 745 | ||
859 | if (!pat_enabled) | 746 | if (!pat_enabled) |
860 | return; | 747 | return; |
861 | 748 | ||
749 | /* | ||
750 | * For now, only handle remap_pfn_range() vmas where | ||
751 | * is_linear_pfn_mapping() == TRUE. Handling of | ||
752 | * vm_insert_pfn() is TBD. | ||
753 | */ | ||
862 | if (is_linear_pfn_mapping(vma)) { | 754 | if (is_linear_pfn_mapping(vma)) { |
863 | /* free the whole chunk starting from vm_pgoff */ | 755 | /* free the whole chunk starting from vm_pgoff */ |
864 | paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT; | 756 | paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT; |
865 | free_pfn_range(paddr, vma_size); | 757 | free_pfn_range(paddr, vma_size); |
866 | return; | 758 | return; |
867 | } | 759 | } |
868 | |||
869 | if (size != 0 && size != vma_size) { | ||
870 | /* free page by page, using pfn and size */ | ||
871 | paddr = (resource_size_t)pfn << PAGE_SHIFT; | ||
872 | for (i = 0; i < size; i += PAGE_SIZE) { | ||
873 | paddr = paddr + i; | ||
874 | free_pfn_range(paddr, PAGE_SIZE); | ||
875 | } | ||
876 | } else { | ||
877 | /* free entire vma, page by page, using the pfn from pte */ | ||
878 | for (i = 0; i < vma_size; i += PAGE_SIZE) { | ||
879 | if (follow_phys(vma, vma_start + i, 0, &prot, &paddr)) | ||
880 | continue; | ||
881 | |||
882 | free_pfn_range(paddr, PAGE_SIZE); | ||
883 | } | ||
884 | } | ||
885 | } | 760 | } |
886 | 761 | ||
887 | pgprot_t pgprot_writecombine(pgprot_t prot) | 762 | pgprot_t pgprot_writecombine(pgprot_t prot) |