mm, x86, pat: rework linear pfn-mmap tracking

Replace the generic vma-flag VM_PFN_AT_MMAP with x86-only VM_PAT.

We can toss mapping address from remap_pfn_range() into
track_pfn_vma_new(), and collect all PAT-related logic together in
arch/x86/.

This patch also restores orignal frustration-free is_cow_mapping() check
in remap_pfn_range(), as it was before commit v2.6.28-rc8-88-g3c8bb73
("x86: PAT: store vm_pgoff for all linear_over_vma_region mappings - v3")

is_linear_pfn_mapping() checks can be removed from mm/huge_memory.c,
because it already handled by VM_PFNMAP in VM_NO_THP bit-mask.

[suresh.b.siddha@intel.com: Reset the VM_PAT flag as part of untrack_pfn_vma()]
Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
Cc: Venkatesh Pallipadi <venki@google.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Nick Piggin <npiggin@kernel.dk>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Carsten Otte <cotte@de.ibm.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Eric Paris <eparis@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: James Morris <james.l.morris@oracle.com>
Cc: Jason Baron <jbaron@redhat.com>
Cc: Kentaro Takeda <takedakn@nttdata.co.jp>
Cc: Matt Helsley <matthltc@us.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Robert Richter <robert.richter@amd.com>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Venkatesh Pallipadi <venki@google.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 13 insertions, 32 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 141dbb69509..73cb22ee966 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1655,11 +1655,7 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma)
         if (vma->vm_ops)
                 /* khugepaged not yet working on file or special mappings */
                 return 0;
-        /*
-         * If is_pfn_mapping() is true is_learn_pfn_mapping() must be
-         * true too, verify it here.
-         */
-        VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP);
+        VM_BUG_ON(vma->vm_flags & VM_NO_THP);
         hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
         hend = vma->vm_end & HPAGE_PMD_MASK;
         if (hstart < hend)
@@ -1912,11 +1908,7 @@ static void collapse_huge_page(struct mm_struct *mm,
                 goto out;
         if (is_vma_temporary_stack(vma))
                 goto out;
-        /*
-         * If is_pfn_mapping() is true is_learn_pfn_mapping() must be
-         * true too, verify it here.
-         */
-        VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP);
+        VM_BUG_ON(vma->vm_flags & VM_NO_THP);
 
         pgd = pgd_offset(mm, address);
         if (!pgd_present(*pgd))
@@ -2154,12 +2146,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
                         goto skip;
                 if (is_vma_temporary_stack(vma))
                         goto skip;
-                /*
-                 * If is_pfn_mapping() is true is_learn_pfn_mapping()
-                 * must be true too, verify it here.
-                 */
-                VM_BUG_ON(is_linear_pfn_mapping(vma) ||
-                          vma->vm_flags & VM_NO_THP);
+                VM_BUG_ON(vma->vm_flags & VM_NO_THP);
 
                 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
                 hend = vma->vm_end & HPAGE_PMD_MASK;
diff --git a/mm/memory.c b/mm/memory.c
index 6bef278ad30..655e1429388 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1055,7 +1055,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         if (is_vm_hugetlb_page(vma))
                 return copy_hugetlb_page_range(dst_mm, src_mm, vma);
 
-        if (unlikely(is_pfn_mapping(vma))) {
+        if (unlikely(vma->vm_flags & VM_PFNMAP)) {
                 /*
                  * We do not free on error cases below as remove_vma
                  * gets called on error from higher level routine
@@ -1327,7 +1327,7 @@ static void unmap_single_vma(struct mmu_gather *tlb,
         if (vma->vm_file)
                 uprobe_munmap(vma, start, end);
 
-        if (unlikely(is_pfn_mapping(vma)))
+        if (unlikely(vma->vm_flags & VM_PFNMAP))
                 untrack_pfn(vma, 0, 0);
 
         if (start != end) {
@@ -2299,26 +2299,20 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
          * There's a horrible special case to handle copy-on-write
          * behaviour that some programs depend on. We mark the "original"
          * un-COW'ed pages by matching them up with "vma->vm_pgoff".
+         * See vm_normal_page() for details.
          */
-        if (addr == vma->vm_start && end == vma->vm_end) {
+        if (is_cow_mapping(vma->vm_flags)) {
+                if (addr != vma->vm_start || end != vma->vm_end)
+                        return -EINVAL;
                 vma->vm_pgoff = pfn;
-                vma->vm_flags |= VM_PFN_AT_MMAP;
-        } else if (is_cow_mapping(vma->vm_flags))
+        }
+
+        err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
+        if (err)
                 return -EINVAL;
 
         vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
 
-        err = track_pfn_remap(vma, &prot, pfn, PAGE_ALIGN(size));
-        if (err) {
-                /*
-                 * To indicate that track_pfn related cleanup is not
-                 * needed from higher level routine calling unmap_vmas
-                 */
-                vma->vm_flags &= ~(VM_IO | VM_RESERVED | VM_PFNMAP);
-                vma->vm_flags &= ~VM_PFN_AT_MMAP;
-                return -EINVAL;
-        }
-
         BUG_ON(addr >= end);
         pfn -= addr >> PAGE_SHIFT;
         pgd = pgd_offset(mm, addr);