10 files changed, 115 insertions, 26 deletions

diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
index f4026bae6eed..05f2b4009ccc 100644
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -1,7 +1,7 @@
 /*
  * linux/mm/allocpercpu.c
  *
- * Separated from slab.c August 11, 2006 Christoph Lameter <clameter@sgi.com>
+ * Separated from slab.c August 11, 2006 Christoph Lameter
  */
 #include <linux/mm.h>
 #include <linux/module.h>
diff --git a/mm/bootmem.c b/mm/bootmem.c
index e8fb927392b9..8d9f60e06f62 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -442,15 +442,17 @@ unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
         return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
 }
 
-void __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
+int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
                                  unsigned long size, int flags)
 {
         int ret;
 
         ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
         if (ret < 0)
-                return;
+                return -ENOMEM;
         reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
+
+        return 0;
 }
 
 void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
diff --git a/mm/memory.c b/mm/memory.c
index 19e0ae9beecb..2302d228fe04 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -999,17 +999,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
                 goto no_page_table;
 
         ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
-        if (!ptep)
-                goto out;
 
         pte = *ptep;
         if (!pte_present(pte))
-                goto unlock;
+                goto no_page;
         if ((flags & FOLL_WRITE) && !pte_write(pte))
                 goto unlock;
         page = vm_normal_page(vma, address, pte);
         if (unlikely(!page))
-                goto unlock;
+                goto bad_page;
 
         if (flags & FOLL_GET)
                 get_page(page);
@@ -1024,6 +1022,15 @@ unlock:
 out:
         return page;
 
+bad_page:
+        pte_unmap_unlock(ptep, ptl);
+        return ERR_PTR(-EFAULT);
+
+no_page:
+        pte_unmap_unlock(ptep, ptl);
+        if (!pte_none(pte))
+                return page;
+        /* Fall through to ZERO_PAGE handling */
 no_page_table:
         /*
          * When core dumping an enormous anonymous area that nobody
@@ -1038,6 +1045,26 @@ no_page_table:
         return page;
 }
 
+/* Can we do the FOLL_ANON optimization? */
+static inline int use_zero_page(struct vm_area_struct *vma)
+{
+        /*
+         * We don't want to optimize FOLL_ANON for make_pages_present()
+         * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
+         * we want to get the page from the page tables to make sure
+         * that we serialize and update with any other user of that
+         * mapping.
+         */
+        if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
+                return 0;
+        /*
+         * And if we have a fault or a nopfn routine, it's not an
+         * anonymous region.
+         */
+        return !vma->vm_ops ||
+                (!vma->vm_ops->fault && !vma->vm_ops->nopfn);
+}
+
 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                 unsigned long start, int len, int write, int force,
                 struct page **pages, struct vm_area_struct **vmas)
@@ -1112,8 +1139,7 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                 foll_flags = FOLL_TOUCH;
                 if (pages)
                         foll_flags |= FOLL_GET;
-                if (!write && !(vma->vm_flags & VM_LOCKED) &&
-                    (!vma->vm_ops || !vma->vm_ops->fault))
+                if (!write && use_zero_page(vma))
                         foll_flags |= FOLL_ANON;
 
                 do {
@@ -1125,7 +1151,7 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                          * be processed until returning to user space.
                          */
                         if (unlikely(test_tsk_thread_flag(tsk, TIF_MEMDIE)))
-                                return -ENOMEM;
+                                return i ? i : -ENOMEM;
 
                         if (write)
                                 foll_flags |= FOLL_WRITE;
@@ -1159,6 +1185,8 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 
                                 cond_resched();
                         }
+                        if (IS_ERR(page))
+                                return i ? i : PTR_ERR(page);
                         if (pages) {
                                 pages[i] = page;
 
@@ -1669,8 +1697,19 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         struct page *dirty_page = NULL;
 
         old_page = vm_normal_page(vma, address, orig_pte);
-        if (!old_page)
+        if (!old_page) {
+                /*
+                 * VM_MIXEDMAP !pfn_valid() case
+                 *
+                 * We should not cow pages in a shared writeable mapping.
+                 * Just mark the pages writable as we can't do any dirty
+                 * accounting on raw pfn maps.
+                 */
+                if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
+                                     (VM_WRITE|VM_SHARED))
+                        goto reuse;
                 goto gotten;
+        }
 
         /*
          * Take out anonymous pages first, anonymous shared vmas are
@@ -1723,6 +1762,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         }
 
         if (reuse) {
+reuse:
                 flush_cache_page(vma, address, pte_pfn(orig_pte));
                 entry = pte_mkyoung(orig_pte);
                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
@@ -1757,7 +1797,6 @@ gotten:
         page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
         if (likely(pte_same(*page_table, orig_pte))) {
                 if (old_page) {
-                        page_remove_rmap(old_page, vma);
                         if (!PageAnon(old_page)) {
                                 dec_mm_counter(mm, file_rss);
                                 inc_mm_counter(mm, anon_rss);
@@ -1779,6 +1818,32 @@ gotten:
                 lru_cache_add_active(new_page);
                 page_add_new_anon_rmap(new_page, vma, address);
 
+                if (old_page) {
+                        /*
+                         * Only after switching the pte to the new page may
+                         * we remove the mapcount here. Otherwise another
+                         * process may come and find the rmap count decremented
+                         * before the pte is switched to the new page, and
+                         * "reuse" the old page writing into it while our pte
+                         * here still points into it and can be read by other
+                         * threads.
+                         *
+                         * The critical issue is to order this
+                         * page_remove_rmap with the ptp_clear_flush above.
+                         * Those stores are ordered by (if nothing else,)
+                         * the barrier present in the atomic_add_negative
+                         * in page_remove_rmap.
+                         *
+                         * Then the TLB flush in ptep_clear_flush ensures that
+                         * no process can access the old page before the
+                         * decremented mapcount is visible. And the old page
+                         * cannot be reused until after the decremented
+                         * mapcount is visible. So transitively, TLBs to
+                         * old page will be flushed before it can be reused.
+                         */
+                        page_remove_rmap(old_page, vma);
+                }
+
                 /* Free the old page.. */
                 new_page = old_page;
                 ret |= VM_FAULT_WRITE;
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index a37a5034f63d..c94e58b192c3 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -729,7 +729,11 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
         } else {
                 *policy = pol == &default_policy ? MPOL_DEFAULT :
                                                 pol->mode;
-                *policy |= pol->flags;
+                /*
+                 * Internal mempolicy flags must be masked off before exposing
+                 * the policy to userspace.
+                 */
+                *policy |= (pol->flags & MPOL_MODE_FLAGS);
         }
 
         if (vma) {
diff --git a/mm/migrate.c b/mm/migrate.c
index 449d77d409f5..55bd355d170d 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -9,7 +9,7 @@
  * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  * Hirokazu Takahashi <taka@valinux.co.jp>
  * Dave Hansen <haveblue@us.ibm.com>
- * Christoph Lameter <clameter@sgi.com>
+ * Christoph Lameter
  */
 
 #include <linux/migrate.h>
@@ -865,6 +865,11 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
                         goto set_status;
 
                 page = follow_page(vma, pp->addr, FOLL_GET);
+
+                err = PTR_ERR(page);
+                if (IS_ERR(page))
+                        goto set_status;
+
                 err = -ENOENT;
                 if (!page)
                         goto set_status;
@@ -928,6 +933,11 @@ static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
                         goto set_status;
 
                 page = follow_page(vma, pm->addr, 0);
+
+                err = PTR_ERR(page);
+                if (IS_ERR(page))
+                        goto set_status;
+
                 err = -ENOENT;
                 /* Use PageReserved to check for zero page */
                 if (!page || PageReserved(page))
diff --git a/mm/mprotect.c b/mm/mprotect.c
index a5bf31c27375..acfe7c8d72fc 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -47,19 +47,17 @@ static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
                 if (pte_present(oldpte)) {
                         pte_t ptent;
 
-                        /* Avoid an SMP race with hardware updated dirty/clean
-                         * bits by wiping the pte and then setting the new pte
-                         * into place.
-                         */
-                        ptent = ptep_get_and_clear(mm, addr, pte);
+                        ptent = ptep_modify_prot_start(mm, addr, pte);
                         ptent = pte_modify(ptent, newprot);
+
                         /*
                          * Avoid taking write faults for pages we know to be
                          * dirty.
                          */
                         if (dirty_accountable && pte_dirty(ptent))
                                 ptent = pte_mkwrite(ptent);
-                        set_pte_at(mm, addr, pte, ptent);
+
+                        ptep_modify_prot_commit(mm, addr, pte, ptent);
 #ifdef CONFIG_MIGRATION
                 } else if (!pte_file(oldpte)) {
                         swp_entry_t entry = pte_to_swp_entry(oldpte);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2f552955a02f..f32fae3121f0 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2328,7 +2328,6 @@ static void build_zonelists(pg_data_t *pgdat)
 static void build_zonelist_cache(pg_data_t *pgdat)
 {
         pgdat->node_zonelists[0].zlcache_ptr = NULL;
-        pgdat->node_zonelists[1].zlcache_ptr = NULL;
 }
 
 #endif  /* CONFIG_NUMA */
diff --git a/mm/slab.c b/mm/slab.c
index 06236e4ddc1b..046607f05f3e 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3263,9 +3263,12 @@ retry:
 
                 if (cpuset_zone_allowed_hardwall(zone, flags) &&
                         cache->nodelists[nid] &&
-                        cache->nodelists[nid]->free_objects)
+                        cache->nodelists[nid]->free_objects) {
                                 obj = ____cache_alloc_node(cache,
                                         flags | GFP_THISNODE, nid);
+                                if (obj)
+                                        break;
+                }
         }
 
         if (!obj) {
diff --git a/mm/slub.c b/mm/slub.c
index 0987d1cd943c..1a427c0ae83b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5,7 +5,7 @@
  * The allocator synchronizes using per slab locks and only
  * uses a centralized lock to manage a pool of partial slabs.
  *
- * (C) 2007 SGI, Christoph Lameter <clameter@sgi.com>
+ * (C) 2007 SGI, Christoph Lameter
  */
 
 #include <linux/mm.h>
@@ -2995,8 +2995,6 @@ void __init kmem_cache_init(void)
                 create_kmalloc_cache(&kmalloc_caches[1],
                                 "kmalloc-96", 96, GFP_KERNEL);
                 caches++;
-        }
-        if (KMALLOC_MIN_SIZE <= 128) {
                 create_kmalloc_cache(&kmalloc_caches[2],
                                 "kmalloc-192", 192, GFP_KERNEL);
                 caches++;
@@ -3026,6 +3024,16 @@ void __init kmem_cache_init(void)
         for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
                 size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;
 
+        if (KMALLOC_MIN_SIZE == 128) {
+                /*
+                 * The 192 byte sized cache is not used if the alignment
+                 * is 128 byte. Redirect kmalloc to use the 256 byte cache
+                 * instead.
+                 */
+                for (i = 128 + 8; i <= 192; i += 8)
+                        size_index[(i - 1) / 8] = 8;
+        }
+
         slab_state = UP;
 
         /* Provide the correct kmalloc names now that the caches are up */
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 99c4f36eb8a3..a91b5f8fcaf6 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -1,7 +1,7 @@
 /*
  * Virtual Memory Map support
  *
- * (C) 2007 sgi. Christoph Lameter <clameter@sgi.com>.
+ * (C) 2007 sgi. Christoph Lameter.
  *
  * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
  * virt_to_page, page_address() to be implemented as a base offset