1 files changed, 58 insertions, 32 deletions

diff --git a/mm/memory.c b/mm/memory.c
index cae514e7dcfc..85e7a87da79f 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -438,7 +438,7 @@ static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
         pgtable_t token = pmd_pgtable(*pmd);
         pmd_clear(pmd);
         pte_free_tlb(tlb, token, addr);
-        atomic_long_dec(&tlb->mm->nr_ptes);
+        mm_dec_nr_ptes(tlb->mm);
 }
 
 static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
@@ -506,6 +506,7 @@ static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
         pud = pud_offset(p4d, start);
         p4d_clear(p4d);
         pud_free_tlb(tlb, pud, start);
+        mm_dec_nr_puds(tlb->mm);
 }
 
 static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
@@ -665,7 +666,7 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
 
         ptl = pmd_lock(mm, pmd);
         if (likely(pmd_none(*pmd))) {   /* Has another populated it ? */
-                atomic_long_inc(&mm->nr_ptes);
+                mm_inc_nr_ptes(mm);
                 pmd_populate(mm, pmd, new);
                 new = NULL;
         }
@@ -2554,7 +2555,11 @@ static int wp_page_copy(struct vm_fault *vmf)
                 put_page(new_page);
 
         pte_unmap_unlock(vmf->pte, vmf->ptl);
-        mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
+        /*
+         * No need to double call mmu_notifier->invalidate_range() callback as
+         * the above ptep_clear_flush_notify() did already call it.
+         */
+        mmu_notifier_invalidate_range_only_end(mm, mmun_start, mmun_end);
         if (old_page) {
                 /*
                  * Don't let another task, with possibly unlocked vma,
@@ -2842,7 +2847,7 @@ EXPORT_SYMBOL(unmap_mapping_range);
 int do_swap_page(struct vm_fault *vmf)
 {
         struct vm_area_struct *vma = vmf->vma;
-        struct page *page = NULL, *swapcache;
+        struct page *page = NULL, *swapcache = NULL;
         struct mem_cgroup *memcg;
         struct vma_swap_readahead swap_ra;
         swp_entry_t entry;
@@ -2881,17 +2886,36 @@ int do_swap_page(struct vm_fault *vmf)
                 }
                 goto out;
         }
+
+
         delayacct_set_flag(DELAYACCT_PF_SWAPIN);
         if (!page)
                 page = lookup_swap_cache(entry, vma_readahead ? vma : NULL,
                                          vmf->address);
         if (!page) {
-                if (vma_readahead)
-                        page = do_swap_page_readahead(entry,
-                                GFP_HIGHUSER_MOVABLE, vmf, &swap_ra);
-                else
-                        page = swapin_readahead(entry,
-                                GFP_HIGHUSER_MOVABLE, vma, vmf->address);
+                struct swap_info_struct *si = swp_swap_info(entry);
+
+                if (si->flags & SWP_SYNCHRONOUS_IO &&
+                                __swap_count(si, entry) == 1) {
+                        /* skip swapcache */
+                        page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
+                        if (page) {
+                                __SetPageLocked(page);
+                                __SetPageSwapBacked(page);
+                                set_page_private(page, entry.val);
+                                lru_cache_add_anon(page);
+                                swap_readpage(page, true);
+                        }
+                } else {
+                        if (vma_readahead)
+                                page = do_swap_page_readahead(entry,
+                                        GFP_HIGHUSER_MOVABLE, vmf, &swap_ra);
+                        else
+                                page = swapin_readahead(entry,
+                                       GFP_HIGHUSER_MOVABLE, vma, vmf->address);
+                        swapcache = page;
+                }
+
                 if (!page) {
                         /*
                          * Back out if somebody else faulted in this pte
@@ -2920,7 +2944,6 @@ int do_swap_page(struct vm_fault *vmf)
                 goto out_release;
         }
 
-        swapcache = page;
         locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
 
         delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
@@ -2935,7 +2958,8 @@ int do_swap_page(struct vm_fault *vmf)
          * test below, are not enough to exclude that.  Even if it is still
          * swapcache, we need to check that the page's swap has not changed.
          */
-        if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val))
+        if (unlikely((!PageSwapCache(page) ||
+                        page_private(page) != entry.val)) && swapcache)
                 goto out_page;
 
         page = ksm_might_need_to_copy(page, vma, vmf->address);
@@ -2988,14 +3012,16 @@ int do_swap_page(struct vm_fault *vmf)
                 pte = pte_mksoft_dirty(pte);
         set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
         vmf->orig_pte = pte;
-        if (page == swapcache) {
-                do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
-                mem_cgroup_commit_charge(page, memcg, true, false);
-                activate_page(page);
-        } else { /* ksm created a completely new copy */
+
+        /* ksm created a completely new copy */
+        if (unlikely(page != swapcache && swapcache)) {
                 page_add_new_anon_rmap(page, vma, vmf->address, false);
                 mem_cgroup_commit_charge(page, memcg, false, false);
                 lru_cache_add_active_or_unevictable(page, vma);
+        } else {
+                do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
+                mem_cgroup_commit_charge(page, memcg, true, false);
+                activate_page(page);
         }
 
         swap_free(entry);
@@ -3003,7 +3029,7 @@ int do_swap_page(struct vm_fault *vmf)
             (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
                 try_to_free_swap(page);
         unlock_page(page);
-        if (page != swapcache) {
+        if (page != swapcache && swapcache) {
                 /*
                  * Hold the lock to avoid the swap entry to be reused
                  * until we take the PT lock for the pte_same() check
@@ -3036,7 +3062,7 @@ out_page:
         unlock_page(page);
 out_release:
         put_page(page);
-        if (page != swapcache) {
+        if (page != swapcache && swapcache) {
                 unlock_page(swapcache);
                 put_page(swapcache);
         }
@@ -3212,7 +3238,7 @@ static int pte_alloc_one_map(struct vm_fault *vmf)
                         goto map_pte;
                 }
 
-                atomic_long_inc(&vma->vm_mm->nr_ptes);
+                mm_inc_nr_ptes(vma->vm_mm);
                 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
                 spin_unlock(vmf->ptl);
                 vmf->prealloc_pte = NULL;
@@ -3271,7 +3297,7 @@ static void deposit_prealloc_pte(struct vm_fault *vmf)
          * We are going to consume the prealloc table,
          * count that as nr_ptes.
          */
-        atomic_long_inc(&vma->vm_mm->nr_ptes);
+        mm_inc_nr_ptes(vma->vm_mm);
         vmf->prealloc_pte = NULL;
 }
 
@@ -4124,15 +4150,17 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
 
         spin_lock(&mm->page_table_lock);
 #ifndef __ARCH_HAS_5LEVEL_HACK
-        if (p4d_present(*p4d))          /* Another has populated it */
-                pud_free(mm, new);
-        else
+        if (!p4d_present(*p4d)) {
+                mm_inc_nr_puds(mm);
                 p4d_populate(mm, p4d, new);
-#else
-        if (pgd_present(*p4d))          /* Another has populated it */
+        } else  /* Another has populated it */
                 pud_free(mm, new);
-        else
+#else
+        if (!pgd_present(*p4d)) {
+                mm_inc_nr_puds(mm);
                 pgd_populate(mm, p4d, new);
+        } else  /* Another has populated it */
+                pud_free(mm, new);
 #endif /* __ARCH_HAS_5LEVEL_HACK */
         spin_unlock(&mm->page_table_lock);
         return 0;
@@ -4457,17 +4485,15 @@ void print_vma_addr(char *prefix, unsigned long ip)
         struct vm_area_struct *vma;
 
         /*
-         * Do not print if we are in atomic
-         * contexts (in exception stacks, etc.):
+         * we might be running from an atomic context so we cannot sleep
          */
-        if (preempt_count())
+        if (!down_read_trylock(&mm->mmap_sem))
                 return;
 
-        down_read(&mm->mmap_sem);
         vma = find_vma(mm, ip);
         if (vma && vma->vm_file) {
                 struct file *f = vma->vm_file;
-                char *buf = (char *)__get_free_page(GFP_KERNEL);
+                char *buf = (char *)__get_free_page(GFP_NOWAIT);
                 if (buf) {
                         char *p;