mm: optimize put_mems_allowed() usage

Since put_mems_allowed() is strictly optional, its a seqcount retry, we
don't need to evaluate the function if the allocation was in fact
successful, saving a smp_rmb some loads and comparisons on some relative
fast-paths.

Since the naming, get/put_mems_allowed() does suggest a mandatory
pairing, rename the interface, as suggested by Mel, to resemble the
seqcount interface.

This gives us: read_mems_allowed_begin() and read_mems_allowed_retry(),
where it is important to note that the return value of the latter call
is inverted from its previous incarnation.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 6 insertions, 6 deletions

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 4755c8576942..e3ab02822799 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1899,7 +1899,7 @@ int node_random(const nodemask_t *maskp)
  * If the effective policy is 'BIND, returns a pointer to the mempolicy's
  * @nodemask for filtering the zonelist.
  *
- * Must be protected by get_mems_allowed()
+ * Must be protected by read_mems_allowed_begin()
  */
 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
                                 gfp_t gfp_flags, struct mempolicy **mpol,
@@ -2063,7 +2063,7 @@ alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
 
 retry_cpuset:
         pol = get_vma_policy(current, vma, addr);
-        cpuset_mems_cookie = get_mems_allowed();
+        cpuset_mems_cookie = read_mems_allowed_begin();
 
         if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
                 unsigned nid;
@@ -2071,7 +2071,7 @@ retry_cpuset:
                 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
                 mpol_cond_put(pol);
                 page = alloc_page_interleave(gfp, order, nid);
-                if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+                if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
                         goto retry_cpuset;
 
                 return page;
@@ -2081,7 +2081,7 @@ retry_cpuset:
                                       policy_nodemask(gfp, pol));
         if (unlikely(mpol_needs_cond_ref(pol)))
                 __mpol_put(pol);
-        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+        if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
                 goto retry_cpuset;
         return page;
 }
@@ -2115,7 +2115,7 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
                 pol = &default_policy;
 
 retry_cpuset:
-        cpuset_mems_cookie = get_mems_allowed();
+        cpuset_mems_cookie = read_mems_allowed_begin();
 
         /*
          * No reference counting needed for current->mempolicy
@@ -2128,7 +2128,7 @@ retry_cpuset:
                                 policy_zonelist(gfp, pol, numa_node_id()),
                                 policy_nodemask(gfp, pol));
 
-        if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+        if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
                 goto retry_cpuset;
 
         return page;