31 files changed, 706 insertions, 111 deletions

diff --git a/mm/Kconfig b/mm/Kconfig
index efee5d379df4..0bd9c2dbb2a0 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -77,9 +77,6 @@ config FLAT_NODE_MEM_MAP
         def_bool y
         depends on !SPARSEMEM
 
-config HAVE_GET_USER_PAGES_FAST
-        bool
-
 #
 # Both the NUMA code and DISCONTIGMEM use arrays of pg_data_t's
 # to represent different areas of memory.  This variable allows
@@ -208,3 +205,6 @@ config NR_QUICK
 config VIRT_TO_BUS
         def_bool y
         depends on !ARCH_NO_VIRT_TO_BUS
+
+config MMU_NOTIFIER
+        bool
diff --git a/mm/Makefile b/mm/Makefile
index 06ca2381fef1..da4ccf015aea 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -25,6 +25,7 @@ obj-$(CONFIG_SHMEM) += shmem.o
 obj-$(CONFIG_TMPFS_POSIX_ACL) += shmem_acl.o
 obj-$(CONFIG_TINY_SHMEM) += tiny-shmem.o
 obj-$(CONFIG_SLOB) += slob.o
+obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
 obj-$(CONFIG_SLAB) += slab.o
 obj-$(CONFIG_SLUB) += slub.o
 obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
diff --git a/mm/filemap.c b/mm/filemap.c
index 5de7633e1dbe..54e968650855 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -558,14 +558,14 @@ EXPORT_SYMBOL(wait_on_page_bit);
  * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
  *
  * The first mb is necessary to safely close the critical section opened by the
- * TestSetPageLocked(), the second mb is necessary to enforce ordering between
- * the clear_bit and the read of the waitqueue (to avoid SMP races with a
- * parallel wait_on_page_locked()).
+ * test_and_set_bit() to lock the page; the second mb is necessary to enforce
+ * ordering between the clear_bit and the read of the waitqueue (to avoid SMP
+ * races with a parallel wait_on_page_locked()).
  */
 void unlock_page(struct page *page)
 {
         smp_mb__before_clear_bit();
-        if (!TestClearPageLocked(page))
+        if (!test_and_clear_bit(PG_locked, &page->flags))
                 BUG();
         smp_mb__after_clear_bit(); 
         wake_up_page(page, PG_locked);
@@ -931,7 +931,7 @@ grab_cache_page_nowait(struct address_space *mapping, pgoff_t index)
         struct page *page = find_get_page(mapping, index);
 
         if (page) {
-                if (!TestSetPageLocked(page))
+                if (trylock_page(page))
                         return page;
                 page_cache_release(page);
                 return NULL;
@@ -1023,8 +1023,17 @@ find_page:
                                         ra, filp, page,
                                         index, last_index - index);
                 }
-                if (!PageUptodate(page))
-                        goto page_not_up_to_date;
+                if (!PageUptodate(page)) {
+                        if (inode->i_blkbits == PAGE_CACHE_SHIFT ||
+                                        !mapping->a_ops->is_partially_uptodate)
+                                goto page_not_up_to_date;
+                        if (!trylock_page(page))
+                                goto page_not_up_to_date;
+                        if (!mapping->a_ops->is_partially_uptodate(page,
+                                                                desc, offset))
+                                goto page_not_up_to_date_locked;
+                        unlock_page(page);
+                }
 page_ok:
                 /*
                  * i_size must be checked after we know the page is Uptodate.
@@ -1094,6 +1103,7 @@ page_not_up_to_date:
                 if (lock_page_killable(page))
                         goto readpage_eio;
 
+page_not_up_to_date_locked:
                 /* Did it get truncated before we got the lock? */
                 if (!page->mapping) {
                         unlock_page(page);
@@ -1869,7 +1879,7 @@ void iov_iter_advance(struct iov_iter *i, size_t bytes)
                  * The !iov->iov_len check ensures we skip over unlikely
                  * zero-length segments (without overruning the iovec).
                  */
-                while (bytes || unlikely(!iov->iov_len && i->count)) {
+                while (bytes || unlikely(i->count && !iov->iov_len)) {
                         int copy;
 
                         copy = min(bytes, iov->iov_len - base);
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c
index 98a3f31ccd6a..380ab402d711 100644
--- a/mm/filemap_xip.c
+++ b/mm/filemap_xip.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/uio.h>
 #include <linux/rmap.h>
+#include <linux/mmu_notifier.h>
 #include <linux/sched.h>
 #include <asm/tlbflush.h>
 #include <asm/io.h>
@@ -188,7 +189,7 @@ __xip_unmap (struct address_space * mapping,
                 if (pte) {
                         /* Nuke the page table entry. */
                         flush_cache_page(vma, address, pte_pfn(*pte));
-                        pteval = ptep_clear_flush(vma, address, pte);
+                        pteval = ptep_clear_flush_notify(vma, address, pte);
                         page_remove_rmap(page, vma);
                         dec_mm_counter(mm, file_rss);
                         BUG_ON(pte_dirty(pteval));
diff --git a/mm/fremap.c b/mm/fremap.c
index 07a9c82ce1a3..7881638e4a12 100644
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -15,6 +15,7 @@
 #include <linux/rmap.h>
 #include <linux/module.h>
 #include <linux/syscalls.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/mmu_context.h>
 #include <asm/cacheflush.h>
@@ -214,7 +215,9 @@ asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size,
                 spin_unlock(&mapping->i_mmap_lock);
         }
 
+        mmu_notifier_invalidate_range_start(mm, start, start + size);
         err = populate_range(mm, vma, start, size, pgoff);
+        mmu_notifier_invalidate_range_end(mm, start, start + size);
         if (!err && !(flags & MAP_NONBLOCK)) {
                 if (unlikely(has_write_lock)) {
                         downgrade_write(&mm->mmap_sem);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 3be79dc18c5c..67a71191136e 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -9,6 +9,7 @@
 #include <linux/mm.h>
 #include <linux/sysctl.h>
 #include <linux/highmem.h>
+#include <linux/mmu_notifier.h>
 #include <linux/nodemask.h>
 #include <linux/pagemap.h>
 #include <linux/mempolicy.h>
@@ -19,6 +20,7 @@
 
 #include <asm/page.h>
 #include <asm/pgtable.h>
+#include <asm/io.h>
 
 #include <linux/hugetlb.h>
 #include "internal.h"
@@ -563,7 +565,7 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
                 huge_page_order(h));
         if (page) {
                 if (arch_prepare_hugepage(page)) {
-                        __free_pages(page, HUGETLB_PAGE_ORDER);
+                        __free_pages(page, huge_page_order(h));
                         return NULL;
                 }
                 prep_new_huge_page(h, page, nid);
@@ -663,6 +665,11 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,
                                         __GFP_REPEAT|__GFP_NOWARN,
                                         huge_page_order(h));
 
+        if (page && arch_prepare_hugepage(page)) {
+                __free_pages(page, huge_page_order(h));
+                return NULL;
+        }
+
         spin_lock(&hugetlb_lock);
         if (page) {
                 /*
@@ -1281,7 +1288,12 @@ module_exit(hugetlb_exit);
 
 static int __init hugetlb_init(void)
 {
-        BUILD_BUG_ON(HPAGE_SHIFT == 0);
+        /* Some platform decide whether they support huge pages at boot
+         * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
+         * there is no such support
+         */
+        if (HPAGE_SHIFT == 0)
+                return 0;
 
         if (!size_to_hstate(default_hstate_size)) {
                 default_hstate_size = HPAGE_SIZE;
@@ -1672,6 +1684,7 @@ void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
         BUG_ON(start & ~huge_page_mask(h));
         BUG_ON(end & ~huge_page_mask(h));
 
+        mmu_notifier_invalidate_range_start(mm, start, end);
         spin_lock(&mm->page_table_lock);
         for (address = start; address < end; address += sz) {
                 ptep = huge_pte_offset(mm, address);
@@ -1713,6 +1726,7 @@ void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
         }
         spin_unlock(&mm->page_table_lock);
         flush_tlb_range(vma, start, end);
+        mmu_notifier_invalidate_range_end(mm, start, end);
         list_for_each_entry_safe(page, tmp, &page_list, lru) {
                 list_del(&page->lru);
                 put_page(page);
@@ -1928,6 +1942,18 @@ retry:
                         lock_page(page);
         }
 
+        /*
+         * If we are going to COW a private mapping later, we examine the
+         * pending reservations for this page now. This will ensure that
+         * any allocations necessary to record that reservation occur outside
+         * the spinlock.
+         */
+        if (write_access && !(vma->vm_flags & VM_SHARED))
+                if (vma_needs_reservation(h, vma, address) < 0) {
+                        ret = VM_FAULT_OOM;
+                        goto backout_unlocked;
+                }
+
         spin_lock(&mm->page_table_lock);
         size = i_size_read(mapping->host) >> huge_page_shift(h);
         if (idx >= size)
@@ -1953,6 +1979,7 @@ out:
 
 backout:
         spin_unlock(&mm->page_table_lock);
+backout_unlocked:
         unlock_page(page);
         put_page(page);
         goto out;
@@ -1964,6 +1991,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         pte_t *ptep;
         pte_t entry;
         int ret;
+        struct page *pagecache_page = NULL;
         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
         struct hstate *h = hstate_vma(vma);
 
@@ -1980,25 +2008,44 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         entry = huge_ptep_get(ptep);
         if (huge_pte_none(entry)) {
                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
-                mutex_unlock(&hugetlb_instantiation_mutex);
-                return ret;
+                goto out_unlock;
         }
 
         ret = 0;
 
+        /*
+         * If we are going to COW the mapping later, we examine the pending
+         * reservations for this page now. This will ensure that any
+         * allocations necessary to record that reservation occur outside the
+         * spinlock. For private mappings, we also lookup the pagecache
+         * page now as it is used to determine if a reservation has been
+         * consumed.
+         */
+        if (write_access && !pte_write(entry)) {
+                if (vma_needs_reservation(h, vma, address) < 0) {
+                        ret = VM_FAULT_OOM;
+                        goto out_unlock;
+                }
+
+                if (!(vma->vm_flags & VM_SHARED))
+                        pagecache_page = hugetlbfs_pagecache_page(h,
+                                                                vma, address);
+        }
+
         spin_lock(&mm->page_table_lock);
         /* Check for a racing update before calling hugetlb_cow */
         if (likely(pte_same(entry, huge_ptep_get(ptep))))
-                if (write_access && !pte_write(entry)) {
-                        struct page *page;
-                        page = hugetlbfs_pagecache_page(h, vma, address);
-                        ret = hugetlb_cow(mm, vma, address, ptep, entry, page);
-                        if (page) {
-                                unlock_page(page);
-                                put_page(page);
-                        }
-                }
+                if (write_access && !pte_write(entry))
+                        ret = hugetlb_cow(mm, vma, address, ptep, entry,
+                                                        pagecache_page);
         spin_unlock(&mm->page_table_lock);
+
+        if (pagecache_page) {
+                unlock_page(pagecache_page);
+                put_page(pagecache_page);
+        }
+
+out_unlock:
         mutex_unlock(&hugetlb_instantiation_mutex);
 
         return ret;
diff --git a/mm/madvise.c b/mm/madvise.c
index 23a0ec3e0ea0..f9349c18a1b5 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -132,10 +132,10 @@ static long madvise_willneed(struct vm_area_struct * vma,
  * Application no longer needs these pages.  If the pages are dirty,
  * it's OK to just throw them away.  The app will be more careful about
  * data it wants to keep.  Be sure to free swap resources too.  The
- * zap_page_range call sets things up for refill_inactive to actually free
+ * zap_page_range call sets things up for shrink_active_list to actually free
  * these pages later if no one else has touched them in the meantime,
  * although we could add these pages to a global reuse list for
- * refill_inactive to pick up before reclaiming other pages.
+ * shrink_active_list to pick up before reclaiming other pages.
  *
  * NB: This interface discards data rather than pushes it out to swap,
  * as some implementations do.  This has performance implications for
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index fba566c51322..0f1f7a7374ba 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -796,6 +796,8 @@ int mem_cgroup_shrink_usage(struct mm_struct *mm, gfp_t gfp_mask)
 
         if (mem_cgroup_subsys.disabled)
                 return 0;
+        if (!mm)
+                return 0;
 
         rcu_read_lock();
         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
@@ -1168,9 +1170,6 @@ static void mem_cgroup_move_task(struct cgroup_subsys *ss,
         mem = mem_cgroup_from_cont(cont);
         old_mem = mem_cgroup_from_cont(old_cont);
 
-        if (mem == old_mem)
-                goto out;
-
         /*
          * Only thread group leaders are allowed to migrate, the mm_struct is
          * in effect owned by the leader
diff --git a/mm/memory.c b/mm/memory.c
index a8ca04faaea6..1002f473f497 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -51,6 +51,7 @@
 #include <linux/init.h>
 #include <linux/writeback.h>
 #include <linux/memcontrol.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>
@@ -652,6 +653,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         unsigned long next;
         unsigned long addr = vma->vm_start;
         unsigned long end = vma->vm_end;
+        int ret;
 
         /*
          * Don't copy ptes where a page fault will fill them correctly.
@@ -667,17 +669,33 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
         if (is_vm_hugetlb_page(vma))
                 return copy_hugetlb_page_range(dst_mm, src_mm, vma);
 
+        /*
+         * We need to invalidate the secondary MMU mappings only when
+         * there could be a permission downgrade on the ptes of the
+         * parent mm. And a permission downgrade will only happen if
+         * is_cow_mapping() returns true.
+         */
+        if (is_cow_mapping(vma->vm_flags))
+                mmu_notifier_invalidate_range_start(src_mm, addr, end);
+
+        ret = 0;
         dst_pgd = pgd_offset(dst_mm, addr);
         src_pgd = pgd_offset(src_mm, addr);
         do {
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(src_pgd))
                         continue;
-                if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
-                                                vma, addr, next))
-                        return -ENOMEM;
+                if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
+                                            vma, addr, next))) {
+                        ret = -ENOMEM;
+                        break;
+                }
         } while (dst_pgd++, src_pgd++, addr = next, addr != end);
-        return 0;
+
+        if (is_cow_mapping(vma->vm_flags))
+                mmu_notifier_invalidate_range_end(src_mm,
+                                                  vma->vm_start, end);
+        return ret;
 }
 
 static unsigned long zap_pte_range(struct mmu_gather *tlb,
@@ -881,7 +899,9 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
         unsigned long start = start_addr;
         spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
         int fullmm = (*tlbp)->fullmm;
+        struct mm_struct *mm = vma->vm_mm;
 
+        mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
         for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
                 unsigned long end;
 
@@ -946,6 +966,7 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
                 }
         }
 out:
+        mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
         return start;   /* which is now the end (or restart) address */
 }
 
@@ -973,6 +994,29 @@ unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
         return end;
 }
 
+/**
+ * zap_vma_ptes - remove ptes mapping the vma
+ * @vma: vm_area_struct holding ptes to be zapped
+ * @address: starting address of pages to zap
+ * @size: number of bytes to zap
+ *
+ * This function only unmaps ptes assigned to VM_PFNMAP vmas.
+ *
+ * The entire address range must be fully contained within the vma.
+ *
+ * Returns 0 if successful.
+ */
+int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
+                unsigned long size)
+{
+        if (address < vma->vm_start || address + size > vma->vm_end ||
+                        !(vma->vm_flags & VM_PFNMAP))
+                return -1;
+        zap_page_range(vma, address, size, NULL);
+        return 0;
+}
+EXPORT_SYMBOL_GPL(zap_vma_ptes);
+
 /*
  * Do a quick page-table lookup for a single page.
  */
@@ -1616,10 +1660,11 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
 {
         pgd_t *pgd;
         unsigned long next;
-        unsigned long end = addr + size;
+        unsigned long start = addr, end = addr + size;
         int err;
 
         BUG_ON(addr >= end);
+        mmu_notifier_invalidate_range_start(mm, start, end);
         pgd = pgd_offset(mm, addr);
         do {
                 next = pgd_addr_end(addr, end);
@@ -1627,6 +1672,7 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
                 if (err)
                         break;
         } while (pgd++, addr = next, addr != end);
+        mmu_notifier_invalidate_range_end(mm, start, end);
         return err;
 }
 EXPORT_SYMBOL_GPL(apply_to_page_range);
@@ -1743,7 +1789,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
          * not dirty accountable.
          */
         if (PageAnon(old_page)) {
-                if (!TestSetPageLocked(old_page)) {
+                if (trylock_page(old_page)) {
                         reuse = can_share_swap_page(old_page);
                         unlock_page(old_page);
                 }
@@ -1839,7 +1885,7 @@ gotten:
                  * seen in the presence of one thread doing SMC and another
                  * thread doing COW.
                  */
-                ptep_clear_flush(vma, address, page_table);
+                ptep_clear_flush_notify(vma, address, page_table);
                 set_pte_at(mm, address, page_table, entry);
                 update_mmu_cache(vma, address, entry);
                 lru_cache_add_active(new_page);
@@ -2719,16 +2765,26 @@ int make_pages_present(unsigned long addr, unsigned long end)
 
         vma = find_vma(current->mm, addr);
         if (!vma)
-                return -1;
+                return -ENOMEM;
         write = (vma->vm_flags & VM_WRITE) != 0;
         BUG_ON(addr >= end);
         BUG_ON(end > vma->vm_end);
         len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE;
         ret = get_user_pages(current, current->mm, addr,
                         len, write, 0, NULL, NULL);
-        if (ret < 0)
+        if (ret < 0) {
+                /*
+                   SUS require strange return value to mlock
+                    - invalid addr generate to ENOMEM.
+                    - out of memory should generate EAGAIN.
+                */
+                if (ret == -EFAULT)
+                        ret = -ENOMEM;
+                else if (ret == -ENOMEM)
+                        ret = -EAGAIN;
                 return ret;
-        return ret == len ? 0 : -1;
+        }
+        return ret == len ? 0 : -ENOMEM;
 }
 
 #if !defined(__HAVE_ARCH_GATE_AREA)
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index e550bec20582..83369058ec13 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -803,7 +803,6 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
 int do_migrate_pages(struct mm_struct *mm,
         const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
 {
-        LIST_HEAD(pagelist);
         int busy = 0;
         int err = 0;
         nodemask_t tmp;
diff --git a/mm/migrate.c b/mm/migrate.c
index 153572fb60b8..2a80136b23bb 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -605,7 +605,7 @@ static int move_to_new_page(struct page *newpage, struct page *page)
          * establishing additional references. We are the only one
          * holding a reference to the new page at this point.
          */
-        if (TestSetPageLocked(newpage))
+        if (!trylock_page(newpage))
                 BUG();
 
         /* Prepare mapping for the new page.*/
@@ -667,7 +667,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
         BUG_ON(charge);
 
         rc = -EAGAIN;
-        if (TestSetPageLocked(page)) {
+        if (!trylock_page(page)) {
                 if (!force)
                         goto move_newpage;
                 lock_page(page);
diff --git a/mm/mlock.c b/mm/mlock.c
index 7b2656055d6a..01fbe93eff5c 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -78,8 +78,6 @@ success:
 
         mm->locked_vm -= pages;
 out:
-        if (ret == -ENOMEM)
-                ret = -EAGAIN;
         return ret;
 }
 
diff --git a/mm/mm_init.c b/mm/mm_init.c
index c6af41ea9994..936ef2efd892 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -14,6 +14,10 @@
 #ifdef CONFIG_DEBUG_MEMORY_INIT
 int __meminitdata mminit_loglevel;
 
+#ifndef SECTIONS_SHIFT
+#define SECTIONS_SHIFT  0
+#endif
+
 /* The zonelists are simply reported, validation is manual. */
 void mminit_verify_zonelist(void)
 {
@@ -74,11 +78,7 @@ void __init mminit_verify_pageflags_layout(void)
                 NR_PAGEFLAGS);
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
                 "Section %d Node %d Zone %d\n",
-#ifdef SECTIONS_SHIFT
                 SECTIONS_SHIFT,
-#else
-                0,
-#endif
                 NODES_SHIFT,
                 ZONES_SHIFT);
         mminit_dprintk(MMINIT_TRACE, "pageflags_layout_offsets",
diff --git a/mm/mmap.c b/mm/mmap.c
index 5e0cc99e9cd5..339cf5c4d5d8 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -26,6 +26,7 @@
 #include <linux/mount.h>
 #include <linux/mempolicy.h>
 #include <linux/rmap.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -369,7 +370,7 @@ find_vma_prepare(struct mm_struct *mm, unsigned long addr,
                 if (vma_tmp->vm_end > addr) {
                         vma = vma_tmp;
                         if (vma_tmp->vm_start <= addr)
-                                return vma;
+                                break;
                         __rb_link = &__rb_parent->rb_left;
                 } else {
                         rb_prev = __rb_parent;
@@ -2061,6 +2062,7 @@ void exit_mmap(struct mm_struct *mm)
 
         /* mm's last user has gone, and its about to be pulled down */
         arch_exit_mmap(mm);
+        mmu_notifier_release(mm);
 
         lru_add_drain();
         flush_cache_mm(mm);
@@ -2268,3 +2270,167 @@ int install_special_mapping(struct mm_struct *mm,
 
         return 0;
 }
+
+static DEFINE_MUTEX(mm_all_locks_mutex);
+
+static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
+{
+        if (!test_bit(0, (unsigned long *) &anon_vma->head.next)) {
+                /*
+                 * The LSB of head.next can't change from under us
+                 * because we hold the mm_all_locks_mutex.
+                 */
+                spin_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
+                /*
+                 * We can safely modify head.next after taking the
+                 * anon_vma->lock. If some other vma in this mm shares
+                 * the same anon_vma we won't take it again.
+                 *
+                 * No need of atomic instructions here, head.next
+                 * can't change from under us thanks to the
+                 * anon_vma->lock.
+                 */
+                if (__test_and_set_bit(0, (unsigned long *)
+                                       &anon_vma->head.next))
+                        BUG();
+        }
+}
+
+static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
+{
+        if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
+                /*
+                 * AS_MM_ALL_LOCKS can't change from under us because
+                 * we hold the mm_all_locks_mutex.
+                 *
+                 * Operations on ->flags have to be atomic because
+                 * even if AS_MM_ALL_LOCKS is stable thanks to the
+                 * mm_all_locks_mutex, there may be other cpus
+                 * changing other bitflags in parallel to us.
+                 */
+                if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
+                        BUG();
+                spin_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
+        }
+}
+
+/*
+ * This operation locks against the VM for all pte/vma/mm related
+ * operations that could ever happen on a certain mm. This includes
+ * vmtruncate, try_to_unmap, and all page faults.
+ *
+ * The caller must take the mmap_sem in write mode before calling
+ * mm_take_all_locks(). The caller isn't allowed to release the
+ * mmap_sem until mm_drop_all_locks() returns.
+ *
+ * mmap_sem in write mode is required in order to block all operations
+ * that could modify pagetables and free pages without need of
+ * altering the vma layout (for example populate_range() with
+ * nonlinear vmas). It's also needed in write mode to avoid new
+ * anon_vmas to be associated with existing vmas.
+ *
+ * A single task can't take more than one mm_take_all_locks() in a row
+ * or it would deadlock.
+ *
+ * The LSB in anon_vma->head.next and the AS_MM_ALL_LOCKS bitflag in
+ * mapping->flags avoid to take the same lock twice, if more than one
+ * vma in this mm is backed by the same anon_vma or address_space.
+ *
+ * We can take all the locks in random order because the VM code
+ * taking i_mmap_lock or anon_vma->lock outside the mmap_sem never
+ * takes more than one of them in a row. Secondly we're protected
+ * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
+ *
+ * mm_take_all_locks() and mm_drop_all_locks are expensive operations
+ * that may have to take thousand of locks.
+ *
+ * mm_take_all_locks() can fail if it's interrupted by signals.
+ */
+int mm_take_all_locks(struct mm_struct *mm)
+{
+        struct vm_area_struct *vma;
+        int ret = -EINTR;
+
+        BUG_ON(down_read_trylock(&mm->mmap_sem));
+
+        mutex_lock(&mm_all_locks_mutex);
+
+        for (vma = mm->mmap; vma; vma = vma->vm_next) {
+                if (signal_pending(current))
+                        goto out_unlock;
+                if (vma->vm_file && vma->vm_file->f_mapping)
+                        vm_lock_mapping(mm, vma->vm_file->f_mapping);
+        }
+
+        for (vma = mm->mmap; vma; vma = vma->vm_next) {
+                if (signal_pending(current))
+                        goto out_unlock;
+                if (vma->anon_vma)
+                        vm_lock_anon_vma(mm, vma->anon_vma);
+        }
+
+        ret = 0;
+
+out_unlock:
+        if (ret)
+                mm_drop_all_locks(mm);
+
+        return ret;
+}
+
+static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
+{
+        if (test_bit(0, (unsigned long *) &anon_vma->head.next)) {
+                /*
+                 * The LSB of head.next can't change to 0 from under
+                 * us because we hold the mm_all_locks_mutex.
+                 *
+                 * We must however clear the bitflag before unlocking
+                 * the vma so the users using the anon_vma->head will
+                 * never see our bitflag.
+                 *
+                 * No need of atomic instructions here, head.next
+                 * can't change from under us until we release the
+                 * anon_vma->lock.
+                 */
+                if (!__test_and_clear_bit(0, (unsigned long *)
+                                          &anon_vma->head.next))
+                        BUG();
+                spin_unlock(&anon_vma->lock);
+        }
+}
+
+static void vm_unlock_mapping(struct address_space *mapping)
+{
+        if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
+                /*
+                 * AS_MM_ALL_LOCKS can't change to 0 from under us
+                 * because we hold the mm_all_locks_mutex.
+                 */
+                spin_unlock(&mapping->i_mmap_lock);
+                if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
+                                        &mapping->flags))
+                        BUG();
+        }
+}
+
+/*
+ * The mmap_sem cannot be released by the caller until
+ * mm_drop_all_locks() returns.
+ */
+void mm_drop_all_locks(struct mm_struct *mm)
+{
+        struct vm_area_struct *vma;
+
+        BUG_ON(down_read_trylock(&mm->mmap_sem));
+        BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
+
+        for (vma = mm->mmap; vma; vma = vma->vm_next) {
+                if (vma->anon_vma)
+                        vm_unlock_anon_vma(vma->anon_vma);
+                if (vma->vm_file && vma->vm_file->f_mapping)
+                        vm_unlock_mapping(vma->vm_file->f_mapping);
+        }
+
+        mutex_unlock(&mm_all_locks_mutex);
+}
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
new file mode 100644
index 000000000000..5f4ef0250bee
--- /dev/null
+++ b/mm/mmu_notifier.c
@@ -0,0 +1,277 @@
+/*
+ *  linux/mm/mmu_notifier.c
+ *
+ *  Copyright (C) 2008  Qumranet, Inc.
+ *  Copyright (C) 2008  SGI
+ *             Christoph Lameter <clameter@sgi.com>
+ *
+ *  This work is licensed under the terms of the GNU GPL, version 2. See
+ *  the COPYING file in the top-level directory.
+ */
+
+#include <linux/rculist.h>
+#include <linux/mmu_notifier.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/err.h>
+#include <linux/rcupdate.h>
+#include <linux/sched.h>
+
+/*
+ * This function can't run concurrently against mmu_notifier_register
+ * because mm->mm_users > 0 during mmu_notifier_register and exit_mmap
+ * runs with mm_users == 0. Other tasks may still invoke mmu notifiers
+ * in parallel despite there being no task using this mm any more,
+ * through the vmas outside of the exit_mmap context, such as with
+ * vmtruncate. This serializes against mmu_notifier_unregister with
+ * the mmu_notifier_mm->lock in addition to RCU and it serializes
+ * against the other mmu notifiers with RCU. struct mmu_notifier_mm
+ * can't go away from under us as exit_mmap holds an mm_count pin
+ * itself.
+ */
+void __mmu_notifier_release(struct mm_struct *mm)
+{
+        struct mmu_notifier *mn;
+
+        spin_lock(&mm->mmu_notifier_mm->lock);
+        while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
+                mn = hlist_entry(mm->mmu_notifier_mm->list.first,
+                                 struct mmu_notifier,
+                                 hlist);
+                /*
+                 * We arrived before mmu_notifier_unregister so
+                 * mmu_notifier_unregister will do nothing other than
+                 * to wait ->release to finish and
+                 * mmu_notifier_unregister to return.
+                 */
+                hlist_del_init_rcu(&mn->hlist);
+                /*
+                 * RCU here will block mmu_notifier_unregister until
+                 * ->release returns.
+                 */
+                rcu_read_lock();
+                spin_unlock(&mm->mmu_notifier_mm->lock);
+                /*
+                 * if ->release runs before mmu_notifier_unregister it
+                 * must be handled as it's the only way for the driver
+                 * to flush all existing sptes and stop the driver
+                 * from establishing any more sptes before all the
+                 * pages in the mm are freed.
+                 */
+                if (mn->ops->release)
+                        mn->ops->release(mn, mm);
+                rcu_read_unlock();
+                spin_lock(&mm->mmu_notifier_mm->lock);
+        }
+        spin_unlock(&mm->mmu_notifier_mm->lock);
+
+        /*
+         * synchronize_rcu here prevents mmu_notifier_release to
+         * return to exit_mmap (which would proceed freeing all pages
+         * in the mm) until the ->release method returns, if it was
+         * invoked by mmu_notifier_unregister.
+         *
+         * The mmu_notifier_mm can't go away from under us because one
+         * mm_count is hold by exit_mmap.
+         */
+        synchronize_rcu();
+}
+
+/*
+ * If no young bitflag is supported by the hardware, ->clear_flush_young can
+ * unmap the address and return 1 or 0 depending if the mapping previously
+ * existed or not.
+ */
+int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
+                                        unsigned long address)
+{
+        struct mmu_notifier *mn;
+        struct hlist_node *n;
+        int young = 0;
+
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->clear_flush_young)
+                        young |= mn->ops->clear_flush_young(mn, mm, address);
+        }
+        rcu_read_unlock();
+
+        return young;
+}
+
+void __mmu_notifier_invalidate_page(struct mm_struct *mm,
+                                          unsigned long address)
+{
+        struct mmu_notifier *mn;
+        struct hlist_node *n;
+
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->invalidate_page)
+                        mn->ops->invalidate_page(mn, mm, address);
+        }
+        rcu_read_unlock();
+}
+
+void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
+                                  unsigned long start, unsigned long end)
+{
+        struct mmu_notifier *mn;
+        struct hlist_node *n;
+
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->invalidate_range_start)
+                        mn->ops->invalidate_range_start(mn, mm, start, end);
+        }
+        rcu_read_unlock();
+}
+
+void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
+                                  unsigned long start, unsigned long end)
+{
+        struct mmu_notifier *mn;
+        struct hlist_node *n;
+
+        rcu_read_lock();
+        hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) {
+                if (mn->ops->invalidate_range_end)
+                        mn->ops->invalidate_range_end(mn, mm, start, end);
+        }
+        rcu_read_unlock();
+}
+
+static int do_mmu_notifier_register(struct mmu_notifier *mn,
+                                    struct mm_struct *mm,
+                                    int take_mmap_sem)
+{
+        struct mmu_notifier_mm *mmu_notifier_mm;
+        int ret;
+
+        BUG_ON(atomic_read(&mm->mm_users) <= 0);
+
+        ret = -ENOMEM;
+        mmu_notifier_mm = kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
+        if (unlikely(!mmu_notifier_mm))
+                goto out;
+
+        if (take_mmap_sem)
+                down_write(&mm->mmap_sem);
+        ret = mm_take_all_locks(mm);
+        if (unlikely(ret))
+                goto out_cleanup;
+
+        if (!mm_has_notifiers(mm)) {
+                INIT_HLIST_HEAD(&mmu_notifier_mm->list);
+                spin_lock_init(&mmu_notifier_mm->lock);
+                mm->mmu_notifier_mm = mmu_notifier_mm;
+                mmu_notifier_mm = NULL;
+        }
+        atomic_inc(&mm->mm_count);
+
+        /*
+         * Serialize the update against mmu_notifier_unregister. A
+         * side note: mmu_notifier_release can't run concurrently with
+         * us because we hold the mm_users pin (either implicitly as
+         * current->mm or explicitly with get_task_mm() or similar).
+         * We can't race against any other mmu notifier method either
+         * thanks to mm_take_all_locks().
+         */
+        spin_lock(&mm->mmu_notifier_mm->lock);
+        hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);
+        spin_unlock(&mm->mmu_notifier_mm->lock);
+
+        mm_drop_all_locks(mm);
+out_cleanup:
+        if (take_mmap_sem)
+                up_write(&mm->mmap_sem);
+        /* kfree() does nothing if mmu_notifier_mm is NULL */
+        kfree(mmu_notifier_mm);
+out:
+        BUG_ON(atomic_read(&mm->mm_users) <= 0);
+        return ret;
+}
+
+/*
+ * Must not hold mmap_sem nor any other VM related lock when calling
+ * this registration function. Must also ensure mm_users can't go down
+ * to zero while this runs to avoid races with mmu_notifier_release,
+ * so mm has to be current->mm or the mm should be pinned safely such
+ * as with get_task_mm(). If the mm is not current->mm, the mm_users
+ * pin should be released by calling mmput after mmu_notifier_register
+ * returns. mmu_notifier_unregister must be always called to
+ * unregister the notifier. mm_count is automatically pinned to allow
+ * mmu_notifier_unregister to safely run at any time later, before or
+ * after exit_mmap. ->release will always be called before exit_mmap
+ * frees the pages.
+ */
+int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
+{
+        return do_mmu_notifier_register(mn, mm, 1);
+}
+EXPORT_SYMBOL_GPL(mmu_notifier_register);
+
+/*
+ * Same as mmu_notifier_register but here the caller must hold the
+ * mmap_sem in write mode.
+ */
+int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
+{
+        return do_mmu_notifier_register(mn, mm, 0);
+}
+EXPORT_SYMBOL_GPL(__mmu_notifier_register);
+
+/* this is called after the last mmu_notifier_unregister() returned */
+void __mmu_notifier_mm_destroy(struct mm_struct *mm)
+{
+        BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));
+        kfree(mm->mmu_notifier_mm);
+        mm->mmu_notifier_mm = LIST_POISON1; /* debug */
+}
+
+/*
+ * This releases the mm_count pin automatically and frees the mm
+ * structure if it was the last user of it. It serializes against
+ * running mmu notifiers with RCU and against mmu_notifier_unregister
+ * with the unregister lock + RCU. All sptes must be dropped before
+ * calling mmu_notifier_unregister. ->release or any other notifier
+ * method may be invoked concurrently with mmu_notifier_unregister,
+ * and only after mmu_notifier_unregister returned we're guaranteed
+ * that ->release or any other method can't run anymore.
+ */
+void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)
+{
+        BUG_ON(atomic_read(&mm->mm_count) <= 0);
+
+        spin_lock(&mm->mmu_notifier_mm->lock);
+        if (!hlist_unhashed(&mn->hlist)) {
+                hlist_del_rcu(&mn->hlist);
+
+                /*
+                 * RCU here will force exit_mmap to wait ->release to finish
+                 * before freeing the pages.
+                 */
+                rcu_read_lock();
+                spin_unlock(&mm->mmu_notifier_mm->lock);
+                /*
+                 * exit_mmap will block in mmu_notifier_release to
+                 * guarantee ->release is called before freeing the
+                 * pages.
+                 */
+                if (mn->ops->release)
+                        mn->ops->release(mn, mm);
+                rcu_read_unlock();
+        } else
+                spin_unlock(&mm->mmu_notifier_mm->lock);
+
+        /*
+         * Wait any running method to finish, of course including
+         * ->release if it was run by mmu_notifier_relase instead of us.
+         */
+        synchronize_rcu();
+
+        BUG_ON(atomic_read(&mm->mm_count) <= 0);
+
+        mmdrop(mm);
+}
+EXPORT_SYMBOL_GPL(mmu_notifier_unregister);
diff --git a/mm/mprotect.c b/mm/mprotect.c
index abd645a3b0a0..fded06f923f4 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -21,6 +21,7 @@
 #include <linux/syscalls.h>
 #include <linux/swap.h>
 #include <linux/swapops.h>
+#include <linux/mmu_notifier.h>
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
 #include <asm/cacheflush.h>
@@ -203,10 +204,12 @@ success:
                 dirty_accountable = 1;
         }
 
+        mmu_notifier_invalidate_range_start(mm, start, end);
         if (is_vm_hugetlb_page(vma))
                 hugetlb_change_protection(vma, start, end, vma->vm_page_prot);
         else
                 change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
+        mmu_notifier_invalidate_range_end(mm, start, end);
         vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
         vm_stat_account(mm, newflags, vma->vm_file, nrpages);
         return 0;
diff --git a/mm/mremap.c b/mm/mremap.c
index 08e3c7f2bd15..1a7743923c8c 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -18,6 +18,7 @@
 #include <linux/highmem.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -74,7 +75,11 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
         struct mm_struct *mm = vma->vm_mm;
         pte_t *old_pte, *new_pte, pte;
         spinlock_t *old_ptl, *new_ptl;
+        unsigned long old_start;
 
+        old_start = old_addr;
+        mmu_notifier_invalidate_range_start(vma->vm_mm,
+                                            old_start, old_end);
         if (vma->vm_file) {
                 /*
                  * Subtle point from Rajesh Venkatasubramanian: before
@@ -116,6 +121,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
         pte_unmap_unlock(old_pte - 1, old_ptl);
         if (mapping)
                 spin_unlock(&mapping->i_mmap_lock);
+        mmu_notifier_invalidate_range_end(vma->vm_mm, old_start, old_end);
 }
 
 #define LATENCY_LIMIT   (64 * PAGE_SIZE)
diff --git a/mm/nommu.c b/mm/nommu.c
index 5edccd9c9218..ed75bc962fbe 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -266,6 +266,27 @@ void *vmalloc_node(unsigned long size, int node)
 }
 EXPORT_SYMBOL(vmalloc_node);
 
+#ifndef PAGE_KERNEL_EXEC
+# define PAGE_KERNEL_EXEC PAGE_KERNEL
+#endif
+
+/**
+ *      vmalloc_exec  -  allocate virtually contiguous, executable memory
+ *      @size:          allocation size
+ *
+ *      Kernel-internal function to allocate enough pages to cover @size
+ *      the page level allocator and map them into contiguous and
+ *      executable kernel virtual space.
+ *
+ *      For tight control over page level allocator and protection flags
+ *      use __vmalloc() instead.
+ */
+
+void *vmalloc_exec(unsigned long size)
+{
+        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
+}
+
 /**
  * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
  *      @size:          allocation size
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6da667274df5..af982f7cdb2a 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2372,7 +2372,7 @@ static void build_zonelist_cache(pg_data_t *pgdat)
 
 #endif  /* CONFIG_NUMA */
 
-/* return values int ....just for stop_machine_run() */
+/* return values int ....just for stop_machine() */
 static int __build_all_zonelists(void *dummy)
 {
         int nid;
@@ -2397,7 +2397,7 @@ void build_all_zonelists(void)
         } else {
                 /* we have to stop all cpus to guarantee there is no user
                    of zonelist */
-                stop_machine_run(__build_all_zonelists, NULL, NR_CPUS);
+                stop_machine(__build_all_zonelists, NULL, NULL);
                 /* cpuset refresh routine should be here */
         }
         vm_total_pages = nr_free_pagecache_pages();
@@ -3753,23 +3753,6 @@ unsigned long __init find_min_pfn_with_active_regions(void)
         return find_min_pfn_for_node(MAX_NUMNODES);
 }
 
-/**
- * find_max_pfn_with_active_regions - Find the maximum PFN registered
- *
- * It returns the maximum PFN based on information provided via
- * add_active_range().
- */
-unsigned long __init find_max_pfn_with_active_regions(void)
-{
-        int i;
-        unsigned long max_pfn = 0;
-
-        for (i = 0; i < nr_nodemap_entries; i++)
-                max_pfn = max(max_pfn, early_node_map[i].end_pfn);
-
-        return max_pfn;
-}
-
 /*
  * early_calculate_totalpages()
  * Sum pages in active regions for movable zone.
@@ -4454,7 +4437,7 @@ void *__init alloc_large_system_hash(const char *tablename,
         do {
                 size = bucketsize << log2qty;
                 if (flags & HASH_EARLY)
-                        table = alloc_bootmem(size);
+                        table = alloc_bootmem_nopanic(size);
                 else if (hashdist)
                         table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
                 else {
diff --git a/mm/rmap.c b/mm/rmap.c
index 39ae5a9bf382..1ea4e6fcee77 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -49,6 +49,7 @@
 #include <linux/module.h>
 #include <linux/kallsyms.h>
 #include <linux/memcontrol.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/tlbflush.h>
 
@@ -287,7 +288,7 @@ static int page_referenced_one(struct page *page,
         if (vma->vm_flags & VM_LOCKED) {
                 referenced++;
                 *mapcount = 1;  /* break early from loop */
-        } else if (ptep_clear_flush_young(vma, address, pte))
+        } else if (ptep_clear_flush_young_notify(vma, address, pte))
                 referenced++;
 
         /* Pretend the page is referenced if the task has the
@@ -421,7 +422,7 @@ int page_referenced(struct page *page, int is_locked,
                         referenced += page_referenced_anon(page, mem_cont);
                 else if (is_locked)
                         referenced += page_referenced_file(page, mem_cont);
-                else if (TestSetPageLocked(page))
+                else if (!trylock_page(page))
                         referenced++;
                 else {
                         if (page->mapping)
@@ -457,7 +458,7 @@ static int page_mkclean_one(struct page *page, struct vm_area_struct *vma)
                 pte_t entry;
 
                 flush_cache_page(vma, address, pte_pfn(*pte));
-                entry = ptep_clear_flush(vma, address, pte);
+                entry = ptep_clear_flush_notify(vma, address, pte);
                 entry = pte_wrprotect(entry);
                 entry = pte_mkclean(entry);
                 set_pte_at(mm, address, pte, entry);
@@ -666,7 +667,8 @@ void page_remove_rmap(struct page *page, struct vm_area_struct *vma)
                  * Leaving it set also helps swapoff to reinstate ptes
                  * faster for those pages still in swapcache.
                  */
-                if (page_test_dirty(page)) {
+                if ((!PageAnon(page) || PageSwapCache(page)) &&
+                    page_test_dirty(page)) {
                         page_clear_dirty(page);
                         set_page_dirty(page);
                 }
@@ -705,14 +707,14 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
          * skipped over this mm) then we should reactivate it.
          */
         if (!migration && ((vma->vm_flags & VM_LOCKED) ||
-                        (ptep_clear_flush_young(vma, address, pte)))) {
+                        (ptep_clear_flush_young_notify(vma, address, pte)))) {
                 ret = SWAP_FAIL;
                 goto out_unmap;
         }
 
         /* Nuke the page table entry. */
         flush_cache_page(vma, address, page_to_pfn(page));
-        pteval = ptep_clear_flush(vma, address, pte);
+        pteval = ptep_clear_flush_notify(vma, address, pte);
 
         /* Move the dirty bit to the physical page now the pte is gone. */
         if (pte_dirty(pteval))
@@ -837,12 +839,12 @@ static void try_to_unmap_cluster(unsigned long cursor,
                 page = vm_normal_page(vma, address, *pte);
                 BUG_ON(!page || PageAnon(page));
 
-                if (ptep_clear_flush_young(vma, address, pte))
+                if (ptep_clear_flush_young_notify(vma, address, pte))
                         continue;
 
                 /* Nuke the page table entry. */
                 flush_cache_page(vma, address, pte_pfn(*pte));
-                pteval = ptep_clear_flush(vma, address, pte);
+                pteval = ptep_clear_flush_notify(vma, address, pte);
 
                 /* If nonlinear, store the file page offset in the pte. */
                 if (page->index != linear_page_index(vma, address))
diff --git a/mm/shmem.c b/mm/shmem.c
index 952d361774bb..04fb4f1ab88e 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1265,7 +1265,7 @@ repeat:
                 }
 
                 /* We have to do this with page locked to prevent races */
-                if (TestSetPageLocked(swappage)) {
+                if (!trylock_page(swappage)) {
                         shmem_swp_unmap(entry);
                         spin_unlock(&info->lock);
                         wait_on_page_locked(swappage);
@@ -1329,7 +1329,7 @@ repeat:
                 shmem_swp_unmap(entry);
                 filepage = find_get_page(mapping, idx);
                 if (filepage &&
-                    (!PageUptodate(filepage) || TestSetPageLocked(filepage))) {
+                    (!PageUptodate(filepage) || !trylock_page(filepage))) {
                         spin_unlock(&info->lock);
                         wait_on_page_locked(filepage);
                         page_cache_release(filepage);
@@ -1513,7 +1513,6 @@ shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
                 inode->i_uid = current->fsuid;
                 inode->i_gid = current->fsgid;
                 inode->i_blocks = 0;
-                inode->i_mapping->a_ops = &shmem_aops;
                 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
                 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
                 inode->i_generation = get_seconds();
@@ -1528,6 +1527,7 @@ shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
                         init_special_inode(inode, mode, dev);
                         break;
                 case S_IFREG:
+                        inode->i_mapping->a_ops = &shmem_aops;
                         inode->i_op = &shmem_inode_operations;
                         inode->i_fop = &shmem_file_operations;
                         mpol_shared_policy_init(&info->policy,
@@ -1929,6 +1929,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
                         return error;
                 }
                 unlock_page(page);
+                inode->i_mapping->a_ops = &shmem_aops;
                 inode->i_op = &shmem_symlink_inode_operations;
                 kaddr = kmap_atomic(page, KM_USER0);
                 memcpy(kaddr, symname, len);
diff --git a/mm/slab.c b/mm/slab.c
index 918f04f7fef1..e76eee466886 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -4472,4 +4472,3 @@ size_t ksize(const void *objp)
 
         return obj_size(virt_to_cache(objp));
 }
-EXPORT_SYMBOL(ksize);
diff --git a/mm/slob.c b/mm/slob.c
index d8fbd4d1bfa7..4c82dd41f32e 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -519,7 +519,6 @@ size_t ksize(const void *block)
         else
                 return sp->page.private;
 }
-EXPORT_SYMBOL(ksize);
 
 struct kmem_cache {
         unsigned int size, align;
diff --git a/mm/slub.c b/mm/slub.c
index b7e2cd5d82db..4f5b96149458 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1329,7 +1329,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
                 n = get_node(s, zone_to_nid(zone));
 
                 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
-                                n->nr_partial > MIN_PARTIAL) {
+                                n->nr_partial > n->min_partial) {
                         page = get_partial_node(n);
                         if (page)
                                 return page;
@@ -1381,7 +1381,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
                 slab_unlock(page);
         } else {
                 stat(c, DEACTIVATE_EMPTY);
-                if (n->nr_partial < MIN_PARTIAL) {
+                if (n->nr_partial < n->min_partial) {
                         /*
                          * Adding an empty slab to the partial slabs in order
                          * to avoid page allocator overhead. This slab needs
@@ -1913,9 +1913,21 @@ static void init_kmem_cache_cpu(struct kmem_cache *s,
 #endif
 }
 
-static void init_kmem_cache_node(struct kmem_cache_node *n)
+static void
+init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
 {
         n->nr_partial = 0;
+
+        /*
+         * The larger the object size is, the more pages we want on the partial
+         * list to avoid pounding the page allocator excessively.
+         */
+        n->min_partial = ilog2(s->size);
+        if (n->min_partial < MIN_PARTIAL)
+                n->min_partial = MIN_PARTIAL;
+        else if (n->min_partial > MAX_PARTIAL)
+                n->min_partial = MAX_PARTIAL;
+
         spin_lock_init(&n->list_lock);
         INIT_LIST_HEAD(&n->partial);
 #ifdef CONFIG_SLUB_DEBUG
@@ -2087,7 +2099,7 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
         init_object(kmalloc_caches, n, 1);
         init_tracking(kmalloc_caches, n);
 #endif
-        init_kmem_cache_node(n);
+        init_kmem_cache_node(n, kmalloc_caches);
         inc_slabs_node(kmalloc_caches, node, page->objects);
 
         /*
@@ -2144,7 +2156,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 
                 }
                 s->node[node] = n;
-                init_kmem_cache_node(n);
+                init_kmem_cache_node(n, s);
         }
         return 1;
 }
@@ -2155,7 +2167,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 
 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 {
-        init_kmem_cache_node(&s->local_node);
+        init_kmem_cache_node(&s->local_node, s);
         return 1;
 }
 #endif
@@ -2715,7 +2727,6 @@ size_t ksize(const void *object)
          */
         return s->size;
 }
-EXPORT_SYMBOL(ksize);
 
 void kfree(const void *x)
 {
@@ -2890,7 +2901,7 @@ static int slab_mem_going_online_callback(void *arg)
                         ret = -ENOMEM;
                         goto out;
                 }
-                init_kmem_cache_node(n);
+                init_kmem_cache_node(n, s);
                 s->node[nid] = n;
         }
 out:
diff --git a/mm/sparse.c b/mm/sparse.c
index 5d9dbbb9d39e..39db301b920d 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -12,7 +12,6 @@
 #include <asm/dma.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
-#include "internal.h"
 
 /*
  * Permanent SPARSEMEM data:
diff --git a/mm/swap.c b/mm/swap.c
index dd89234ee51f..9e0cb3118079 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -278,9 +278,10 @@ int lru_add_drain_all(void)
  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
  * for the remainder of the operation.
  *
- * The locking in this function is against shrink_cache(): we recheck the
- * page count inside the lock to see whether shrink_cache grabbed the page
- * via the LRU.  If it did, give up: shrink_cache will free it.
+ * The locking in this function is against shrink_inactive_list(): we recheck
+ * the page count inside the lock to see whether shrink_inactive_list()
+ * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
+ * will free it.
  */
 void release_pages(struct page **pages, int nr, int cold)
 {
@@ -443,7 +444,7 @@ void pagevec_strip(struct pagevec *pvec)
         for (i = 0; i < pagevec_count(pvec); i++) {
                 struct page *page = pvec->pages[i];
 
-                if (PagePrivate(page) && !TestSetPageLocked(page)) {
+                if (PagePrivate(page) && trylock_page(page)) {
                         if (PagePrivate(page))
                                 try_to_release_page(page, 0);
                         unlock_page(page);
diff --git a/mm/swap_state.c b/mm/swap_state.c
index b8035b055129..167cf2dc8a03 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -201,7 +201,7 @@ void delete_from_swap_cache(struct page *page)
  */
 static inline void free_swap_cache(struct page *page)
 {
-        if (PageSwapCache(page) && !TestSetPageLocked(page)) {
+        if (PageSwapCache(page) && trylock_page(page)) {
                 remove_exclusive_swap_page(page);
                 unlock_page(page);
         }
@@ -302,9 +302,9 @@ struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                  * re-using the just freed swap entry for an existing page.
                  * May fail (-ENOMEM) if radix-tree node allocation failed.
                  */
-                SetPageLocked(new_page);
+                set_page_locked(new_page);
                 err = add_to_swap_cache(new_page, entry, gfp_mask & GFP_KERNEL);
-                if (!err) {
+                if (likely(!err)) {
                         /*
                          * Initiate read into locked page and return.
                          */
@@ -312,7 +312,7 @@ struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                         swap_readpage(NULL, new_page);
                         return new_page;
                 }
-                ClearPageLocked(new_page);
+                clear_page_locked(new_page);
                 swap_free(entry);
         } while (err != -ENOMEM);
 
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 6beb6251e99d..1e330f2998fa 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -403,7 +403,7 @@ void free_swap_and_cache(swp_entry_t entry)
         if (p) {
                 if (swap_entry_free(p, swp_offset(entry)) == 1) {
                         page = find_get_page(&swapper_space, entry.val);
-                        if (page && unlikely(TestSetPageLocked(page))) {
+                        if (page && unlikely(!trylock_page(page))) {
                                 page_cache_release(page);
                                 page = NULL;
                         }
@@ -656,8 +656,8 @@ static int unuse_mm(struct mm_struct *mm,
 
         if (!down_read_trylock(&mm->mmap_sem)) {
                 /*
-                 * Activate page so shrink_cache is unlikely to unmap its
-                 * ptes while lock is dropped, so swapoff can make progress.
+                 * Activate page so shrink_inactive_list is unlikely to unmap
+                 * its ptes while lock is dropped, so swapoff can make progress.
                  */
                 activate_page(page);
                 unlock_page(page);
diff --git a/mm/truncate.c b/mm/truncate.c
index e68443d74567..250505091d37 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -104,7 +104,6 @@ truncate_complete_page(struct address_space *mapping, struct page *page)
         cancel_dirty_page(page, PAGE_CACHE_SIZE);
 
         remove_from_page_cache(page);
-        ClearPageUptodate(page);
         ClearPageMappedToDisk(page);
         page_cache_release(page);       /* pagecache ref */
 }
@@ -188,7 +187,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
                         if (page_index > next)
                                 next = page_index;
                         next++;
-                        if (TestSetPageLocked(page))
+                        if (!trylock_page(page))
                                 continue;
                         if (PageWriteback(page)) {
                                 unlock_page(page);
@@ -281,7 +280,7 @@ unsigned long __invalidate_mapping_pages(struct address_space *mapping,
                         pgoff_t index;
                         int lock_failed;
 
-                        lock_failed = TestSetPageLocked(page);
+                        lock_failed = !trylock_page(page);
 
                         /*
                          * We really shouldn't be looking at the ->index of an
@@ -356,7 +355,6 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
         BUG_ON(PagePrivate(page));
         __remove_from_page_cache(page);
         spin_unlock_irq(&mapping->tree_lock);
-        ClearPageUptodate(page);
         page_cache_release(page);       /* pagecache ref */
         return 1;
 failed:
diff --git a/mm/util.c b/mm/util.c
index 9341ca77bd88..cb00b748ce47 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -171,3 +171,18 @@ void arch_pick_mmap_layout(struct mm_struct *mm)
         mm->unmap_area = arch_unmap_area;
 }
 #endif
+
+int __attribute__((weak)) get_user_pages_fast(unsigned long start,
+                                int nr_pages, int write, struct page **pages)
+{
+        struct mm_struct *mm = current->mm;
+        int ret;
+
+        down_read(&mm->mmap_sem);
+        ret = get_user_pages(current, mm, start, nr_pages,
+                                        write, 0, pages, NULL);
+        up_read(&mm->mmap_sem);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(get_user_pages_fast);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 8f71761bc4b7..1ff1a58e7c10 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -496,7 +496,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                 page = lru_to_page(page_list);
                 list_del(&page->lru);
 
-                if (TestSetPageLocked(page))
+                if (!trylock_page(page))
                         goto keep;
 
                 VM_BUG_ON(PageActive(page));
@@ -582,7 +582,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                                  * A synchronous write - probably a ramdisk.  Go
                                  * ahead and try to reclaim the page.
                                  */
-                                if (TestSetPageLocked(page))
+                                if (!trylock_page(page))
                                         goto keep;
                                 if (PageDirty(page) || PageWriteback(page))
                                         goto keep_locked;
@@ -1408,7 +1408,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
                 if (sc->nr_scanned && priority < DEF_PRIORITY - 2)
                         congestion_wait(WRITE, HZ/10);
         }
-        /* top priority shrink_caches still had more to do? don't OOM, then */
+        /* top priority shrink_zones still had more to do? don't OOM, then */
         if (!sc->all_unreclaimable && scan_global_lru(sc))
                 ret = nr_reclaimed;
 out:
@@ -1979,7 +1979,7 @@ module_init(kswapd_init)
 int zone_reclaim_mode __read_mostly;
 
 #define RECLAIM_OFF 0
-#define RECLAIM_ZONE (1<<0)     /* Run shrink_cache on the zone */
+#define RECLAIM_ZONE (1<<0)     /* Run shrink_inactive_list on the zone */
 #define RECLAIM_WRITE (1<<1)    /* Writeout pages during reclaim */
 #define RECLAIM_SWAP (1<<2)     /* Swap pages out during reclaim */