7 files changed, 95 insertions, 47 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index f3230604006c..843042045dc9 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -499,10 +499,13 @@ struct page *__page_cache_alloc(gfp_t gfp)
         struct page *page;
 
         if (cpuset_do_page_mem_spread()) {
-                get_mems_allowed();
-                n = cpuset_mem_spread_node();
-                page = alloc_pages_exact_node(n, gfp, 0);
-                put_mems_allowed();
+                unsigned int cpuset_mems_cookie;
+                do {
+                        cpuset_mems_cookie = get_mems_allowed();
+                        n = cpuset_mem_spread_node();
+                        page = alloc_pages_exact_node(n, gfp, 0);
+                } while (!put_mems_allowed(cpuset_mems_cookie) && !page);
+
                 return page;
         }
         return alloc_pages(gfp, 0);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 62f9fada4d6d..b1c314877334 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -454,14 +454,16 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
                                 struct vm_area_struct *vma,
                                 unsigned long address, int avoid_reserve)
 {
-        struct page *page = NULL;
+        struct page *page;
         struct mempolicy *mpol;
         nodemask_t *nodemask;
         struct zonelist *zonelist;
         struct zone *zone;
         struct zoneref *z;
+        unsigned int cpuset_mems_cookie;
 
-        get_mems_allowed();
+retry_cpuset:
+        cpuset_mems_cookie = get_mems_allowed();
         zonelist = huge_zonelist(vma, address,
                                         htlb_alloc_mask, &mpol, &nodemask);
         /*
@@ -488,10 +490,15 @@ static struct page *dequeue_huge_page_vma(struct hstate *h,
                         }
                 }
         }
-err:
+
         mpol_cond_put(mpol);
-        put_mems_allowed();
+        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                goto retry_cpuset;
         return page;
+
+err:
+        mpol_cond_put(mpol);
+        return NULL;
 }
 
 static void update_and_free_page(struct hstate *h, struct page *page)
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 71e1a523e209..cfb6c8678754 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1850,18 +1850,24 @@ struct page *
 alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
                 unsigned long addr, int node)
 {
-        struct mempolicy *pol = get_vma_policy(current, vma, addr);
+        struct mempolicy *pol;
         struct zonelist *zl;
         struct page *page;
+        unsigned int cpuset_mems_cookie;
+
+retry_cpuset:
+        pol = get_vma_policy(current, vma, addr);
+        cpuset_mems_cookie = get_mems_allowed();
 
-        get_mems_allowed();
         if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
                 unsigned nid;
 
                 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
                 mpol_cond_put(pol);
                 page = alloc_page_interleave(gfp, order, nid);
-                put_mems_allowed();
+                if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                        goto retry_cpuset;
+
                 return page;
         }
         zl = policy_zonelist(gfp, pol, node);
@@ -1872,7 +1878,8 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
                 struct page *page =  __alloc_pages_nodemask(gfp, order,
                                                 zl, policy_nodemask(gfp, pol));
                 __mpol_put(pol);
-                put_mems_allowed();
+                if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                        goto retry_cpuset;
                 return page;
         }
         /*
@@ -1880,7 +1887,8 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
          */
         page = __alloc_pages_nodemask(gfp, order, zl,
                                       policy_nodemask(gfp, pol));
-        put_mems_allowed();
+        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                goto retry_cpuset;
         return page;
 }
 
@@ -1907,11 +1915,14 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
         struct mempolicy *pol = current->mempolicy;
         struct page *page;
+        unsigned int cpuset_mems_cookie;
 
         if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
                 pol = &default_policy;
 
-        get_mems_allowed();
+retry_cpuset:
+        cpuset_mems_cookie = get_mems_allowed();
+
         /*
          * No reference counting needed for current->mempolicy
          * nor system default_policy
@@ -1922,7 +1933,10 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
                 page = __alloc_pages_nodemask(gfp, order,
                                 policy_zonelist(gfp, pol, numa_node_id()),
                                 policy_nodemask(gfp, pol));
-        put_mems_allowed();
+
+        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                goto retry_cpuset;
+
         return page;
 }
 EXPORT_SYMBOL(alloc_pages_current);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 673596ad9c80..40de6854b980 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2380,8 +2380,9 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
 {
         enum zone_type high_zoneidx = gfp_zone(gfp_mask);
         struct zone *preferred_zone;
-        struct page *page;
+        struct page *page = NULL;
         int migratetype = allocflags_to_migratetype(gfp_mask);
+        unsigned int cpuset_mems_cookie;
 
         gfp_mask &= gfp_allowed_mask;
 
@@ -2400,15 +2401,15 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
         if (unlikely(!zonelist->_zonerefs->zone))
                 return NULL;
 
-        get_mems_allowed();
+retry_cpuset:
+        cpuset_mems_cookie = get_mems_allowed();
+
         /* The preferred zone is used for statistics later */
         first_zones_zonelist(zonelist, high_zoneidx,
                                 nodemask ? : &cpuset_current_mems_allowed,
                                 &preferred_zone);
-        if (!preferred_zone) {
-                put_mems_allowed();
-                return NULL;
-        }
+        if (!preferred_zone)
+                goto out;
 
         /* First allocation attempt */
         page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
@@ -2418,9 +2419,19 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
                 page = __alloc_pages_slowpath(gfp_mask, order,
                                 zonelist, high_zoneidx, nodemask,
                                 preferred_zone, migratetype);
-        put_mems_allowed();
 
         trace_mm_page_alloc(page, order, gfp_mask, migratetype);
+
+out:
+        /*
+         * When updating a task's mems_allowed, it is possible to race with
+         * parallel threads in such a way that an allocation can fail while
+         * the mask is being updated. If a page allocation is about to fail,
+         * check if the cpuset changed during allocation and if so, retry.
+         */
+        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                goto retry_cpuset;
+
         return page;
 }
 EXPORT_SYMBOL(__alloc_pages_nodemask);
@@ -2634,13 +2645,15 @@ void si_meminfo_node(struct sysinfo *val, int nid)
 bool skip_free_areas_node(unsigned int flags, int nid)
 {
         bool ret = false;
+        unsigned int cpuset_mems_cookie;
 
         if (!(flags & SHOW_MEM_FILTER_NODES))
                 goto out;
 
-        get_mems_allowed();
-        ret = !node_isset(nid, cpuset_current_mems_allowed);
-        put_mems_allowed();
+        do {
+                cpuset_mems_cookie = get_mems_allowed();
+                ret = !node_isset(nid, cpuset_current_mems_allowed);
+        } while (!put_mems_allowed(cpuset_mems_cookie));
 out:
         return ret;
 }
diff --git a/mm/slab.c b/mm/slab.c
index f0bd7857ab3b..29c8716eb7a9 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3284,12 +3284,10 @@ static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
         if (in_interrupt() || (flags & __GFP_THISNODE))
                 return NULL;
         nid_alloc = nid_here = numa_mem_id();
-        get_mems_allowed();
         if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
                 nid_alloc = cpuset_slab_spread_node();
         else if (current->mempolicy)
                 nid_alloc = slab_node(current->mempolicy);
-        put_mems_allowed();
         if (nid_alloc != nid_here)
                 return ____cache_alloc_node(cachep, flags, nid_alloc);
         return NULL;
@@ -3312,14 +3310,17 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
         enum zone_type high_zoneidx = gfp_zone(flags);
         void *obj = NULL;
         int nid;
+        unsigned int cpuset_mems_cookie;
 
         if (flags & __GFP_THISNODE)
                 return NULL;
 
-        get_mems_allowed();
-        zonelist = node_zonelist(slab_node(current->mempolicy), flags);
         local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
 
+retry_cpuset:
+        cpuset_mems_cookie = get_mems_allowed();
+        zonelist = node_zonelist(slab_node(current->mempolicy), flags);
+
 retry:
         /*
          * Look through allowed nodes for objects available
@@ -3372,7 +3373,9 @@ retry:
                         }
                 }
         }
-        put_mems_allowed();
+
+        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !obj))
+                goto retry_cpuset;
         return obj;
 }
 
diff --git a/mm/slub.c b/mm/slub.c
index 4907563ef7ff..f4a6229848fd 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1581,6 +1581,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags,
         struct zone *zone;
         enum zone_type high_zoneidx = gfp_zone(flags);
         void *object;
+        unsigned int cpuset_mems_cookie;
 
         /*
          * The defrag ratio allows a configuration of the tradeoffs between
@@ -1604,23 +1605,32 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags,
                         get_cycles() % 1024 > s->remote_node_defrag_ratio)
                 return NULL;
 
-        get_mems_allowed();
-        zonelist = node_zonelist(slab_node(current->mempolicy), flags);
-        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
-                struct kmem_cache_node *n;
-
-                n = get_node(s, zone_to_nid(zone));
-
-                if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
-                                n->nr_partial > s->min_partial) {
-                        object = get_partial_node(s, n, c);
-                        if (object) {
-                                put_mems_allowed();
-                                return object;
+        do {
+                cpuset_mems_cookie = get_mems_allowed();
+                zonelist = node_zonelist(slab_node(current->mempolicy), flags);
+                for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
+                        struct kmem_cache_node *n;
+
+                        n = get_node(s, zone_to_nid(zone));
+
+                        if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
+                                        n->nr_partial > s->min_partial) {
+                                object = get_partial_node(s, n, c);
+                                if (object) {
+                                        /*
+                                         * Return the object even if
+                                         * put_mems_allowed indicated that
+                                         * the cpuset mems_allowed was
+                                         * updated in parallel. It's a
+                                         * harmless race between the alloc
+                                         * and the cpuset update.
+                                         */
+                                        put_mems_allowed(cpuset_mems_cookie);
+                                        return object;
+                                }
                         }
                 }
-        }
-        put_mems_allowed();
+        } while (!put_mems_allowed(cpuset_mems_cookie));
 #endif
         return NULL;
 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 440af1d899b9..55d86c9506f3 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2343,7 +2343,6 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
         unsigned long writeback_threshold;
         bool aborted_reclaim;
 
-        get_mems_allowed();
         delayacct_freepages_start();
 
         if (global_reclaim(sc))
@@ -2407,7 +2406,6 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
 
 out:
         delayacct_freepages_end();
-        put_mems_allowed();
 
         if (sc->nr_reclaimed)
                 return sc->nr_reclaimed;