1 files changed, 149 insertions, 79 deletions

diff --git a/mm/memory.c b/mm/memory.c
index 0d14d1e58a5f..bbab1e37055e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -371,57 +371,93 @@ static inline int is_cow_mapping(unsigned int flags)
 }
 
 /*
- * This function gets the "struct page" associated with a pte.
+ * vm_normal_page -- This function gets the "struct page" associated with a pte.
  *
- * NOTE! Some mappings do not have "struct pages". A raw PFN mapping
- * will have each page table entry just pointing to a raw page frame
- * number, and as far as the VM layer is concerned, those do not have
- * pages associated with them - even if the PFN might point to memory
- * that otherwise is perfectly fine and has a "struct page".
+ * "Special" mappings do not wish to be associated with a "struct page" (either
+ * it doesn't exist, or it exists but they don't want to touch it). In this
+ * case, NULL is returned here. "Normal" mappings do have a struct page.
  *
- * The way we recognize those mappings is through the rules set up
- * by "remap_pfn_range()": the vma will have the VM_PFNMAP bit set,
- * and the vm_pgoff will point to the first PFN mapped: thus every
- * page that is a raw mapping will always honor the rule
+ * There are 2 broad cases. Firstly, an architecture may define a pte_special()
+ * pte bit, in which case this function is trivial. Secondly, an architecture
+ * may not have a spare pte bit, which requires a more complicated scheme,
+ * described below.
+ *
+ * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
+ * special mapping (even if there are underlying and valid "struct pages").
+ * COWed pages of a VM_PFNMAP are always normal.
+ *
+ * The way we recognize COWed pages within VM_PFNMAP mappings is through the
+ * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
+ * set, and the vm_pgoff will point to the first PFN mapped: thus every special
+ * mapping will always honor the rule
  *
  *      pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
  *
- * and if that isn't true, the page has been COW'ed (in which case it
- * _does_ have a "struct page" associated with it even if it is in a
- * VM_PFNMAP range).
+ * And for normal mappings this is false.
+ *
+ * This restricts such mappings to be a linear translation from virtual address
+ * to pfn. To get around this restriction, we allow arbitrary mappings so long
+ * as the vma is not a COW mapping; in that case, we know that all ptes are
+ * special (because none can have been COWed).
+ *
+ *
+ * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
+ *
+ * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
+ * page" backing, however the difference is that _all_ pages with a struct
+ * page (that is, those where pfn_valid is true) are refcounted and considered
+ * normal pages by the VM. The disadvantage is that pages are refcounted
+ * (which can be slower and simply not an option for some PFNMAP users). The
+ * advantage is that we don't have to follow the strict linearity rule of
+ * PFNMAP mappings in order to support COWable mappings.
+ *
  */
-struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
+#ifdef __HAVE_ARCH_PTE_SPECIAL
+# define HAVE_PTE_SPECIAL 1
+#else
+# define HAVE_PTE_SPECIAL 0
+#endif
+struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
+                                pte_t pte)
 {
-        unsigned long pfn = pte_pfn(pte);
+        unsigned long pfn;
 
-        if (unlikely(vma->vm_flags & VM_PFNMAP)) {
-                unsigned long off = (addr - vma->vm_start) >> PAGE_SHIFT;
-                if (pfn == vma->vm_pgoff + off)
-                        return NULL;
-                if (!is_cow_mapping(vma->vm_flags))
-                        return NULL;
+        if (HAVE_PTE_SPECIAL) {
+                if (likely(!pte_special(pte))) {
+                        VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
+                        return pte_page(pte);
+                }
+                VM_BUG_ON(!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
+                return NULL;
         }
 
-#ifdef CONFIG_DEBUG_VM
-        /*
-         * Add some anal sanity checks for now. Eventually,
-         * we should just do "return pfn_to_page(pfn)", but
-         * in the meantime we check that we get a valid pfn,
-         * and that the resulting page looks ok.
-         */
-        if (unlikely(!pfn_valid(pfn))) {
-                print_bad_pte(vma, pte, addr);
-                return NULL;
+        /* !HAVE_PTE_SPECIAL case follows: */
+
+        pfn = pte_pfn(pte);
+
+        if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
+                if (vma->vm_flags & VM_MIXEDMAP) {
+                        if (!pfn_valid(pfn))
+                                return NULL;
+                        goto out;
+                } else {
+                        unsigned long off;
+                        off = (addr - vma->vm_start) >> PAGE_SHIFT;
+                        if (pfn == vma->vm_pgoff + off)
+                                return NULL;
+                        if (!is_cow_mapping(vma->vm_flags))
+                                return NULL;
+                }
         }
-#endif
+
+        VM_BUG_ON(!pfn_valid(pfn));
 
         /*
-         * NOTE! We still have PageReserved() pages in the page 
-         * tables. 
+         * NOTE! We still have PageReserved() pages in the page tables.
          *
-         * The PAGE_ZERO() pages and various VDSO mappings can
-         * cause them to exist.
+         * eg. VDSO mappings can cause them to exist.
          */
+out:
         return pfn_to_page(pfn);
 }
 
@@ -1057,8 +1093,7 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                 if (pages)
                         foll_flags |= FOLL_GET;
                 if (!write && !(vma->vm_flags & VM_LOCKED) &&
-                    (!vma->vm_ops || (!vma->vm_ops->nopage &&
-                                        !vma->vm_ops->fault)))
+                    (!vma->vm_ops || !vma->vm_ops->fault))
                         foll_flags |= FOLL_ANON;
 
                 do {
@@ -1141,8 +1176,10 @@ pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
  * old drivers should use this, and they needed to mark their
  * pages reserved for the old functions anyway.
  */
-static int insert_page(struct mm_struct *mm, unsigned long addr, struct page *page, pgprot_t prot)
+static int insert_page(struct vm_area_struct *vma, unsigned long addr,
+                        struct page *page, pgprot_t prot)
 {
+        struct mm_struct *mm = vma->vm_mm;
         int retval;
         pte_t *pte;
         spinlock_t *ptl;
@@ -1202,40 +1239,26 @@ out:
  *
  * The page does not need to be reserved.
  */
-int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, struct page *page)
+int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
+                        struct page *page)
 {
         if (addr < vma->vm_start || addr >= vma->vm_end)
                 return -EFAULT;
         if (!page_count(page))
                 return -EINVAL;
         vma->vm_flags |= VM_INSERTPAGE;
-        return insert_page(vma->vm_mm, addr, page, vma->vm_page_prot);
+        return insert_page(vma, addr, page, vma->vm_page_prot);
 }
 EXPORT_SYMBOL(vm_insert_page);
 
-/**
- * vm_insert_pfn - insert single pfn into user vma
- * @vma: user vma to map to
- * @addr: target user address of this page
- * @pfn: source kernel pfn
- *
- * Similar to vm_inert_page, this allows drivers to insert individual pages
- * they've allocated into a user vma. Same comments apply.
- *
- * This function should only be called from a vm_ops->fault handler, and
- * in that case the handler should return NULL.
- */
-int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
-                unsigned long pfn)
+static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
+                        unsigned long pfn, pgprot_t prot)
 {
         struct mm_struct *mm = vma->vm_mm;
         int retval;
         pte_t *pte, entry;
         spinlock_t *ptl;
 
-        BUG_ON(!(vma->vm_flags & VM_PFNMAP));
-        BUG_ON(is_cow_mapping(vma->vm_flags));
-
         retval = -ENOMEM;
         pte = get_locked_pte(mm, addr, &ptl);
         if (!pte)
@@ -1245,19 +1268,74 @@ int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                 goto out_unlock;
 
         /* Ok, finally just insert the thing.. */
-        entry = pfn_pte(pfn, vma->vm_page_prot);
+        entry = pte_mkspecial(pfn_pte(pfn, prot));
         set_pte_at(mm, addr, pte, entry);
-        update_mmu_cache(vma, addr, entry);
+        update_mmu_cache(vma, addr, entry); /* XXX: why not for insert_page? */
 
         retval = 0;
 out_unlock:
         pte_unmap_unlock(pte, ptl);
-
 out:
         return retval;
 }
+
+/**
+ * vm_insert_pfn - insert single pfn into user vma
+ * @vma: user vma to map to
+ * @addr: target user address of this page
+ * @pfn: source kernel pfn
+ *
+ * Similar to vm_inert_page, this allows drivers to insert individual pages
+ * they've allocated into a user vma. Same comments apply.
+ *
+ * This function should only be called from a vm_ops->fault handler, and
+ * in that case the handler should return NULL.
+ */
+int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
+                        unsigned long pfn)
+{
+        /*
+         * Technically, architectures with pte_special can avoid all these
+         * restrictions (same for remap_pfn_range).  However we would like
+         * consistency in testing and feature parity among all, so we should
+         * try to keep these invariants in place for everybody.
+         */
+        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
+        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
+                                                (VM_PFNMAP|VM_MIXEDMAP));
+        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
+        BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
+
+        if (addr < vma->vm_start || addr >= vma->vm_end)
+                return -EFAULT;
+        return insert_pfn(vma, addr, pfn, vma->vm_page_prot);
+}
 EXPORT_SYMBOL(vm_insert_pfn);
 
+int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
+                        unsigned long pfn)
+{
+        BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
+
+        if (addr < vma->vm_start || addr >= vma->vm_end)
+                return -EFAULT;
+
+        /*
+         * If we don't have pte special, then we have to use the pfn_valid()
+         * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
+         * refcount the page if pfn_valid is true (hence insert_page rather
+         * than insert_pfn).
+         */
+        if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) {
+                struct page *page;
+
+                page = pfn_to_page(pfn);
+                return insert_page(vma, addr, page, vma->vm_page_prot);
+        }
+        return insert_pfn(vma, addr, pfn, vma->vm_page_prot);
+}
+EXPORT_SYMBOL(vm_insert_mixed);
+
 /*
  * maps a range of physical memory into the requested pages. the old
  * mappings are removed. any references to nonexistent pages results
@@ -1276,7 +1354,7 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
         arch_enter_lazy_mmu_mode();
         do {
                 BUG_ON(!pte_none(*pte));
-                set_pte_at(mm, addr, pte, pfn_pte(pfn, prot));
+                set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
                 pfn++;
         } while (pte++, addr += PAGE_SIZE, addr != end);
         arch_leave_lazy_mmu_mode();
@@ -2199,20 +2277,9 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 
         BUG_ON(vma->vm_flags & VM_PFNMAP);
 
-        if (likely(vma->vm_ops->fault)) {
-                ret = vma->vm_ops->fault(vma, &vmf);
-                if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
-                        return ret;
-        } else {
-                /* Legacy ->nopage path */
-                ret = 0;
-                vmf.page = vma->vm_ops->nopage(vma, address & PAGE_MASK, &ret);
-                /* no page was available -- either SIGBUS or OOM */
-                if (unlikely(vmf.page == NOPAGE_SIGBUS))
-                        return VM_FAULT_SIGBUS;
-                else if (unlikely(vmf.page == NOPAGE_OOM))
-                        return VM_FAULT_OOM;
-        }
+        ret = vma->vm_ops->fault(vma, &vmf);
+        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
+                return ret;
 
         /*
          * For consistency in subsequent calls, make the faulted page always
@@ -2377,10 +2444,13 @@ static noinline int do_no_pfn(struct mm_struct *mm, struct vm_area_struct *vma,
         unsigned long pfn;
 
         pte_unmap(page_table);
-        BUG_ON(!(vma->vm_flags & VM_PFNMAP));
-        BUG_ON(is_cow_mapping(vma->vm_flags));
+        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
+        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
 
         pfn = vma->vm_ops->nopfn(vma, address & PAGE_MASK);
+
+        BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
+
         if (unlikely(pfn == NOPFN_OOM))
                 return VM_FAULT_OOM;
         else if (unlikely(pfn == NOPFN_SIGBUS))
@@ -2458,7 +2528,7 @@ static inline int handle_pte_fault(struct mm_struct *mm,
         if (!pte_present(entry)) {
                 if (pte_none(entry)) {
                         if (vma->vm_ops) {
-                                if (vma->vm_ops->fault || vma->vm_ops->nopage)
+                                if (likely(vma->vm_ops->fault))
                                         return do_linear_fault(mm, vma, address,
                                                 pte, pmd, write_access, entry);
                                 if (unlikely(vma->vm_ops->nopfn))