1 files changed, 145 insertions, 59 deletions

diff --git a/mm/memory.c b/mm/memory.c
index 7b9db658aca2..e009ce870859 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -52,6 +52,9 @@
 #include <linux/writeback.h>
 #include <linux/memcontrol.h>
 #include <linux/mmu_notifier.h>
+#include <linux/kallsyms.h>
+#include <linux/swapops.h>
+#include <linux/elf.h>
 
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>
@@ -59,9 +62,6 @@
 #include <asm/tlbflush.h>
 #include <asm/pgtable.h>
 
-#include <linux/swapops.h>
-#include <linux/elf.h>
-
 #include "internal.h"
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
@@ -375,15 +375,65 @@ static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
  *
  * The calling function must still handle the error.
  */
-static void print_bad_pte(struct vm_area_struct *vma, pte_t pte,
-                          unsigned long vaddr)
-{
-        printk(KERN_ERR "Bad pte = %08llx, process = %s, "
-                        "vm_flags = %lx, vaddr = %lx\n",
-                (long long)pte_val(pte),
-                (vma->vm_mm == current->mm ? current->comm : "???"),
-                vma->vm_flags, vaddr);
+static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
+                          pte_t pte, struct page *page)
+{
+        pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
+        pud_t *pud = pud_offset(pgd, addr);
+        pmd_t *pmd = pmd_offset(pud, addr);
+        struct address_space *mapping;
+        pgoff_t index;
+        static unsigned long resume;
+        static unsigned long nr_shown;
+        static unsigned long nr_unshown;
+
+        /*
+         * Allow a burst of 60 reports, then keep quiet for that minute;
+         * or allow a steady drip of one report per second.
+         */
+        if (nr_shown == 60) {
+                if (time_before(jiffies, resume)) {
+                        nr_unshown++;
+                        return;
+                }
+                if (nr_unshown) {
+                        printk(KERN_ALERT
+                                "BUG: Bad page map: %lu messages suppressed\n",
+                                nr_unshown);
+                        nr_unshown = 0;
+                }
+                nr_shown = 0;
+        }
+        if (nr_shown++ == 0)
+                resume = jiffies + 60 * HZ;
+
+        mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
+        index = linear_page_index(vma, addr);
+
+        printk(KERN_ALERT
+                "BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
+                current->comm,
+                (long long)pte_val(pte), (long long)pmd_val(*pmd));
+        if (page) {
+                printk(KERN_ALERT
+                "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n",
+                page, (void *)page->flags, page_count(page),
+                page_mapcount(page), page->mapping, page->index);
+        }
+        printk(KERN_ALERT
+                "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
+                (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
+        /*
+         * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
+         */
+        if (vma->vm_ops)
+                print_symbol(KERN_ALERT "vma->vm_ops->fault: %s\n",
+                                (unsigned long)vma->vm_ops->fault);
+        if (vma->vm_file && vma->vm_file->f_op)
+                print_symbol(KERN_ALERT "vma->vm_file->f_op->mmap: %s\n",
+                                (unsigned long)vma->vm_file->f_op->mmap);
         dump_stack();
+        add_taint(TAINT_BAD_PAGE);
 }
 
 static inline int is_cow_mapping(unsigned int flags)
@@ -441,21 +491,18 @@ static inline int is_cow_mapping(unsigned int flags)
 struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
                                 pte_t pte)
 {
-        unsigned long pfn;
+        unsigned long pfn = pte_pfn(pte);
 
         if (HAVE_PTE_SPECIAL) {
-                if (likely(!pte_special(pte))) {
-                        VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
-                        return pte_page(pte);
-                }
-                VM_BUG_ON(!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
+                if (likely(!pte_special(pte)))
+                        goto check_pfn;
+                if (!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)))
+                        print_bad_pte(vma, addr, pte, NULL);
                 return NULL;
         }
 
         /* !HAVE_PTE_SPECIAL case follows: */
 
-        pfn = pte_pfn(pte);
-
         if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
                 if (vma->vm_flags & VM_MIXEDMAP) {
                         if (!pfn_valid(pfn))
@@ -471,11 +518,14 @@ struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
                 }
         }
 
-        VM_BUG_ON(!pfn_valid(pfn));
+check_pfn:
+        if (unlikely(pfn > highest_memmap_pfn)) {
+                print_bad_pte(vma, addr, pte, NULL);
+                return NULL;
+        }
 
         /*
          * NOTE! We still have PageReserved() pages in the page tables.
-         *
          * eg. VDSO mappings can cause them to exist.
          */
 out:
@@ -767,11 +817,14 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
                         else {
                                 if (pte_dirty(ptent))
                                         set_page_dirty(page);
-                                if (pte_young(ptent))
-                                        SetPageReferenced(page);
+                                if (pte_young(ptent) &&
+                                    likely(!VM_SequentialReadHint(vma)))
+                                        mark_page_accessed(page);
                                 file_rss--;
                         }
-                        page_remove_rmap(page, vma);
+                        page_remove_rmap(page);
+                        if (unlikely(page_mapcount(page) < 0))
+                                print_bad_pte(vma, addr, ptent, page);
                         tlb_remove_page(tlb, page);
                         continue;
                 }
@@ -781,8 +834,12 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
                  */
                 if (unlikely(details))
                         continue;
-                if (!pte_file(ptent))
-                        free_swap_and_cache(pte_to_swp_entry(ptent));
+                if (pte_file(ptent)) {
+                        if (unlikely(!(vma->vm_flags & VM_NONLINEAR)))
+                                print_bad_pte(vma, addr, ptent, NULL);
+                } else if
+                  (unlikely(!free_swap_and_cache(pte_to_swp_entry(ptent))))
+                        print_bad_pte(vma, addr, ptent, NULL);
                 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
         } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
 
@@ -1153,6 +1210,7 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
         int write = !!(flags & GUP_FLAGS_WRITE);
         int force = !!(flags & GUP_FLAGS_FORCE);
         int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
+        int ignore_sigkill = !!(flags & GUP_FLAGS_IGNORE_SIGKILL);
 
         if (len <= 0)
                 return 0;
@@ -1231,12 +1289,15 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                         struct page *page;
 
                         /*
-                         * If tsk is ooming, cut off its access to large memory
-                         * allocations. It has a pending SIGKILL, but it can't
-                         * be processed until returning to user space.
+                         * If we have a pending SIGKILL, don't keep faulting
+                         * pages and potentially allocating memory, unless
+                         * current is handling munlock--e.g., on exit. In
+                         * that case, we are not allocating memory.  Rather,
+                         * we're only unlocking already resident/mapped pages.
                          */
-                        if (unlikely(test_tsk_thread_flag(tsk, TIF_MEMDIE)))
-                                return i ? i : -ENOMEM;
+                        if (unlikely(!ignore_sigkill &&
+                                        fatal_signal_pending(current)))
+                                return i ? i : -ERESTARTSYS;
 
                         if (write)
                                 foll_flags |= FOLL_WRITE;
@@ -1263,9 +1324,15 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                                  * do_wp_page has broken COW when necessary,
                                  * even if maybe_mkwrite decided not to set
                                  * pte_write. We can thus safely do subsequent
-                                 * page lookups as if they were reads.
+                                 * page lookups as if they were reads. But only
+                                 * do so when looping for pte_write is futile:
+                                 * in some cases userspace may also be wanting
+                                 * to write to the gotten user page, which a
+                                 * read fault here might prevent (a readonly
+                                 * page might get reCOWed by userspace write).
                                  */
-                                if (ret & VM_FAULT_WRITE)
+                                if ((ret & VM_FAULT_WRITE) &&
+                                    !(vma->vm_flags & VM_WRITE))
                                         foll_flags &= ~FOLL_WRITE;
 
                                 cond_resched();
@@ -1644,6 +1711,8 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
 
         BUG_ON(pmd_huge(*pmd));
 
+        arch_enter_lazy_mmu_mode();
+
         token = pmd_pgtable(*pmd);
 
         do {
@@ -1652,6 +1721,8 @@ static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
                         break;
         } while (pte++, addr += PAGE_SIZE, addr != end);
 
+        arch_leave_lazy_mmu_mode();
+
         if (mm != &init_mm)
                 pte_unmap_unlock(pte-1, ptl);
         return err;
@@ -1837,10 +1908,21 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
          * not dirty accountable.
          */
         if (PageAnon(old_page)) {
-                if (trylock_page(old_page)) {
-                        reuse = can_share_swap_page(old_page);
-                        unlock_page(old_page);
+                if (!trylock_page(old_page)) {
+                        page_cache_get(old_page);
+                        pte_unmap_unlock(page_table, ptl);
+                        lock_page(old_page);
+                        page_table = pte_offset_map_lock(mm, pmd, address,
+                                                         &ptl);
+                        if (!pte_same(*page_table, orig_pte)) {
+                                unlock_page(old_page);
+                                page_cache_release(old_page);
+                                goto unlock;
+                        }
+                        page_cache_release(old_page);
                 }
+                reuse = reuse_swap_page(old_page);
+                unlock_page(old_page);
         } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
                                         (VM_WRITE|VM_SHARED))) {
                 /*
@@ -1918,7 +2000,7 @@ gotten:
         cow_user_page(new_page, old_page, address, vma);
         __SetPageUptodate(new_page);
 
-        if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
+        if (mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))
                 goto oom_free_new;
 
         /*
@@ -1943,11 +2025,7 @@ gotten:
                  * thread doing COW.
                  */
                 ptep_clear_flush_notify(vma, address, page_table);
-                SetPageSwapBacked(new_page);
-                lru_cache_add_active_or_unevictable(new_page, vma);
                 page_add_new_anon_rmap(new_page, vma, address);
-
-//TODO:  is this safe?  do_anonymous_page() does it this way.
                 set_pte_at(mm, address, page_table, entry);
                 update_mmu_cache(vma, address, entry);
                 if (old_page) {
@@ -1973,7 +2051,7 @@ gotten:
                          * mapcount is visible. So transitively, TLBs to
                          * old page will be flushed before it can be reused.
                          */
-                        page_remove_rmap(old_page, vma);
+                        page_remove_rmap(old_page);
                 }
 
                 /* Free the old page.. */
@@ -2314,6 +2392,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
         struct page *page;
         swp_entry_t entry;
         pte_t pte;
+        struct mem_cgroup *ptr = NULL;
         int ret = 0;
 
         if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
@@ -2352,7 +2431,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
         lock_page(page);
         delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
 
-        if (mem_cgroup_charge(page, mm, GFP_KERNEL)) {
+        if (mem_cgroup_try_charge_swapin(mm, page, GFP_KERNEL, &ptr)) {
                 ret = VM_FAULT_OOM;
                 unlock_page(page);
                 goto out;
@@ -2370,22 +2449,35 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto out_nomap;
         }
 
-        /* The page isn't present yet, go ahead with the fault. */
+        /*
+         * The page isn't present yet, go ahead with the fault.
+         *
+         * Be careful about the sequence of operations here.
+         * To get its accounting right, reuse_swap_page() must be called
+         * while the page is counted on swap but not yet in mapcount i.e.
+         * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
+         * must be called after the swap_free(), or it will never succeed.
+         * Because delete_from_swap_page() may be called by reuse_swap_page(),
+         * mem_cgroup_commit_charge_swapin() may not be able to find swp_entry
+         * in page->private. In this case, a record in swap_cgroup  is silently
+         * discarded at swap_free().
+         */
 
         inc_mm_counter(mm, anon_rss);
         pte = mk_pte(page, vma->vm_page_prot);
-        if (write_access && can_share_swap_page(page)) {
+        if (write_access && reuse_swap_page(page)) {
                 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
                 write_access = 0;
         }
-
         flush_icache_page(vma, page);
         set_pte_at(mm, address, page_table, pte);
         page_add_anon_rmap(page, vma, address);
+        /* It's better to call commit-charge after rmap is established */
+        mem_cgroup_commit_charge_swapin(page, ptr);
 
         swap_free(entry);
         if (vm_swap_full() || (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
-                remove_exclusive_swap_page(page);
+                try_to_free_swap(page);
         unlock_page(page);
 
         if (write_access) {
@@ -2402,7 +2494,7 @@ unlock:
 out:
         return ret;
 out_nomap:
-        mem_cgroup_uncharge_page(page);
+        mem_cgroup_cancel_charge_swapin(ptr);
         pte_unmap_unlock(page_table, ptl);
         unlock_page(page);
         page_cache_release(page);
@@ -2432,7 +2524,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto oom;
         __SetPageUptodate(page);
 
-        if (mem_cgroup_charge(page, mm, GFP_KERNEL))
+        if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))
                 goto oom_free_page;
 
         entry = mk_pte(page, vma->vm_page_prot);
@@ -2442,8 +2534,6 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
         if (!pte_none(*page_table))
                 goto release;
         inc_mm_counter(mm, anon_rss);
-        SetPageSwapBacked(page);
-        lru_cache_add_active_or_unevictable(page, vma);
         page_add_new_anon_rmap(page, vma, address);
         set_pte_at(mm, address, page_table, entry);
 
@@ -2525,7 +2615,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                                 ret = VM_FAULT_OOM;
                                 goto out;
                         }
-                        if (mem_cgroup_charge(page, mm, GFP_KERNEL)) {
+                        if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) {
                                 ret = VM_FAULT_OOM;
                                 page_cache_release(page);
                                 goto out;
@@ -2591,8 +2681,6 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                         entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                 if (anon) {
                         inc_mm_counter(mm, anon_rss);
-                        SetPageSwapBacked(page);
-                        lru_cache_add_active_or_unevictable(page, vma);
                         page_add_new_anon_rmap(page, vma, address);
                 } else {
                         inc_mm_counter(mm, file_rss);
@@ -2602,7 +2690,6 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                                 get_page(dirty_page);
                         }
                 }
-//TODO:  is this safe?  do_anonymous_page() does it this way.
                 set_pte_at(mm, address, page_table, entry);
 
                 /* no need to invalidate: a not-present page won't be cached */
@@ -2666,12 +2753,11 @@ static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
                 return 0;
 
-        if (unlikely(!(vma->vm_flags & VM_NONLINEAR) ||
-                        !(vma->vm_flags & VM_CAN_NONLINEAR))) {
+        if (unlikely(!(vma->vm_flags & VM_NONLINEAR))) {
                 /*
                  * Page table corrupted: show pte and kill process.
                  */
-                print_bad_pte(vma, orig_pte, address);
+                print_bad_pte(vma, address, orig_pte, NULL);
                 return VM_FAULT_OOM;
         }
 
@@ -2953,7 +3039,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
 {
         resource_size_t phys_addr;
         unsigned long prot = 0;
-        void *maddr;
+        void __iomem *maddr;
         int offset = addr & (PAGE_SIZE-1);
 
         if (follow_phys(vma, addr, write, &prot, &phys_addr))