10 files changed, 87 insertions, 28 deletions

diff --git a/mm/Kconfig b/mm/Kconfig
index 446c6588c753..0bd9c2dbb2a0 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -77,9 +77,6 @@ config FLAT_NODE_MEM_MAP
         def_bool y
         depends on !SPARSEMEM
 
-config HAVE_GET_USER_PAGES_FAST
-        bool
-
 #
 # Both the NUMA code and DISCONTIGMEM use arrays of pg_data_t's
 # to represent different areas of memory.  This variable allows
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 757ca983fd99..67a71191136e 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -565,7 +565,7 @@ static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
                 huge_page_order(h));
         if (page) {
                 if (arch_prepare_hugepage(page)) {
-                        __free_pages(page, HUGETLB_PAGE_ORDER);
+                        __free_pages(page, huge_page_order(h));
                         return NULL;
                 }
                 prep_new_huge_page(h, page, nid);
@@ -665,6 +665,11 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,
                                         __GFP_REPEAT|__GFP_NOWARN,
                                         huge_page_order(h));
 
+        if (page && arch_prepare_hugepage(page)) {
+                __free_pages(page, huge_page_order(h));
+                return NULL;
+        }
+
         spin_lock(&hugetlb_lock);
         if (page) {
                 /*
@@ -1937,6 +1942,18 @@ retry:
                         lock_page(page);
         }
 
+        /*
+         * If we are going to COW a private mapping later, we examine the
+         * pending reservations for this page now. This will ensure that
+         * any allocations necessary to record that reservation occur outside
+         * the spinlock.
+         */
+        if (write_access && !(vma->vm_flags & VM_SHARED))
+                if (vma_needs_reservation(h, vma, address) < 0) {
+                        ret = VM_FAULT_OOM;
+                        goto backout_unlocked;
+                }
+
         spin_lock(&mm->page_table_lock);
         size = i_size_read(mapping->host) >> huge_page_shift(h);
         if (idx >= size)
@@ -1962,6 +1979,7 @@ out:
 
 backout:
         spin_unlock(&mm->page_table_lock);
+backout_unlocked:
         unlock_page(page);
         put_page(page);
         goto out;
@@ -1973,6 +1991,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         pte_t *ptep;
         pte_t entry;
         int ret;
+        struct page *pagecache_page = NULL;
         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
         struct hstate *h = hstate_vma(vma);
 
@@ -1989,25 +2008,44 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         entry = huge_ptep_get(ptep);
         if (huge_pte_none(entry)) {
                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
-                mutex_unlock(&hugetlb_instantiation_mutex);
-                return ret;
+                goto out_unlock;
         }
 
         ret = 0;
 
+        /*
+         * If we are going to COW the mapping later, we examine the pending
+         * reservations for this page now. This will ensure that any
+         * allocations necessary to record that reservation occur outside the
+         * spinlock. For private mappings, we also lookup the pagecache
+         * page now as it is used to determine if a reservation has been
+         * consumed.
+         */
+        if (write_access && !pte_write(entry)) {
+                if (vma_needs_reservation(h, vma, address) < 0) {
+                        ret = VM_FAULT_OOM;
+                        goto out_unlock;
+                }
+
+                if (!(vma->vm_flags & VM_SHARED))
+                        pagecache_page = hugetlbfs_pagecache_page(h,
+                                                                vma, address);
+        }
+
         spin_lock(&mm->page_table_lock);
         /* Check for a racing update before calling hugetlb_cow */
         if (likely(pte_same(entry, huge_ptep_get(ptep))))
-                if (write_access && !pte_write(entry)) {
-                        struct page *page;
-                        page = hugetlbfs_pagecache_page(h, vma, address);
-                        ret = hugetlb_cow(mm, vma, address, ptep, entry, page);
-                        if (page) {
-                                unlock_page(page);
-                                put_page(page);
-                        }
-                }
+                if (write_access && !pte_write(entry))
+                        ret = hugetlb_cow(mm, vma, address, ptep, entry,
+                                                        pagecache_page);
         spin_unlock(&mm->page_table_lock);
+
+        if (pagecache_page) {
+                unlock_page(pagecache_page);
+                put_page(pagecache_page);
+        }
+
+out_unlock:
         mutex_unlock(&hugetlb_instantiation_mutex);
 
         return ret;
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 7056c3bdb478..0f1f7a7374ba 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -796,6 +796,8 @@ int mem_cgroup_shrink_usage(struct mm_struct *mm, gfp_t gfp_mask)
 
         if (mem_cgroup_subsys.disabled)
                 return 0;
+        if (!mm)
+                return 0;
 
         rcu_read_lock();
         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index e550bec20582..83369058ec13 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -803,7 +803,6 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
 int do_migrate_pages(struct mm_struct *mm,
         const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
 {
-        LIST_HEAD(pagelist);
         int busy = 0;
         int err = 0;
         nodemask_t tmp;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 401d104d2bb6..af982f7cdb2a 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4437,7 +4437,7 @@ void *__init alloc_large_system_hash(const char *tablename,
         do {
                 size = bucketsize << log2qty;
                 if (flags & HASH_EARLY)
-                        table = alloc_bootmem(size);
+                        table = alloc_bootmem_nopanic(size);
                 else if (hashdist)
                         table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
                 else {
diff --git a/mm/slab.c b/mm/slab.c
index 918f04f7fef1..e76eee466886 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -4472,4 +4472,3 @@ size_t ksize(const void *objp)
 
         return obj_size(virt_to_cache(objp));
 }
-EXPORT_SYMBOL(ksize);
diff --git a/mm/slob.c b/mm/slob.c
index d8fbd4d1bfa7..4c82dd41f32e 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -519,7 +519,6 @@ size_t ksize(const void *block)
         else
                 return sp->page.private;
 }
-EXPORT_SYMBOL(ksize);
 
 struct kmem_cache {
         unsigned int size, align;
diff --git a/mm/slub.c b/mm/slub.c
index b7e2cd5d82db..4f5b96149458 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1329,7 +1329,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
                 n = get_node(s, zone_to_nid(zone));
 
                 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
-                                n->nr_partial > MIN_PARTIAL) {
+                                n->nr_partial > n->min_partial) {
                         page = get_partial_node(n);
                         if (page)
                                 return page;
@@ -1381,7 +1381,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
                 slab_unlock(page);
         } else {
                 stat(c, DEACTIVATE_EMPTY);
-                if (n->nr_partial < MIN_PARTIAL) {
+                if (n->nr_partial < n->min_partial) {
                         /*
                          * Adding an empty slab to the partial slabs in order
                          * to avoid page allocator overhead. This slab needs
@@ -1913,9 +1913,21 @@ static void init_kmem_cache_cpu(struct kmem_cache *s,
 #endif
 }
 
-static void init_kmem_cache_node(struct kmem_cache_node *n)
+static void
+init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
 {
         n->nr_partial = 0;
+
+        /*
+         * The larger the object size is, the more pages we want on the partial
+         * list to avoid pounding the page allocator excessively.
+         */
+        n->min_partial = ilog2(s->size);
+        if (n->min_partial < MIN_PARTIAL)
+                n->min_partial = MIN_PARTIAL;
+        else if (n->min_partial > MAX_PARTIAL)
+                n->min_partial = MAX_PARTIAL;
+
         spin_lock_init(&n->list_lock);
         INIT_LIST_HEAD(&n->partial);
 #ifdef CONFIG_SLUB_DEBUG
@@ -2087,7 +2099,7 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
         init_object(kmalloc_caches, n, 1);
         init_tracking(kmalloc_caches, n);
 #endif
-        init_kmem_cache_node(n);
+        init_kmem_cache_node(n, kmalloc_caches);
         inc_slabs_node(kmalloc_caches, node, page->objects);
 
         /*
@@ -2144,7 +2156,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 
                 }
                 s->node[node] = n;
-                init_kmem_cache_node(n);
+                init_kmem_cache_node(n, s);
         }
         return 1;
 }
@@ -2155,7 +2167,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 
 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 {
-        init_kmem_cache_node(&s->local_node);
+        init_kmem_cache_node(&s->local_node, s);
         return 1;
 }
 #endif
@@ -2715,7 +2727,6 @@ size_t ksize(const void *object)
          */
         return s->size;
 }
-EXPORT_SYMBOL(ksize);
 
 void kfree(const void *x)
 {
@@ -2890,7 +2901,7 @@ static int slab_mem_going_online_callback(void *arg)
                         ret = -ENOMEM;
                         goto out;
                 }
-                init_kmem_cache_node(n);
+                init_kmem_cache_node(n, s);
                 s->node[nid] = n;
         }
 out:
diff --git a/mm/sparse.c b/mm/sparse.c
index 5d9dbbb9d39e..39db301b920d 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -12,7 +12,6 @@
 #include <asm/dma.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
-#include "internal.h"
 
 /*
  * Permanent SPARSEMEM data:
diff --git a/mm/util.c b/mm/util.c
index 9341ca77bd88..cb00b748ce47 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -171,3 +171,18 @@ void arch_pick_mmap_layout(struct mm_struct *mm)
         mm->unmap_area = arch_unmap_area;
 }
 #endif
+
+int __attribute__((weak)) get_user_pages_fast(unsigned long start,
+                                int nr_pages, int write, struct page **pages)
+{
+        struct mm_struct *mm = current->mm;
+        int ret;
+
+        down_read(&mm->mmap_sem);
+        ret = get_user_pages(current, mm, start, nr_pages,
+                                        write, 0, pages, NULL);
+        up_read(&mm->mmap_sem);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(get_user_pages_fast);