[PATCH] cpuset memory spread slab cache optimizations

The hooks in the slab cache allocator code path for support of NUMA
mempolicies and cpuset memory spreading are in an important code path.  Many
systems will use neither feature.

This patch optimizes those hooks down to a single check of some bits in the
current tasks task_struct flags.  For non NUMA systems, this hook and related
code is already ifdef'd out.

The optimization is done by using another task flag, set if the task is using
a non-default NUMA mempolicy.  Taking this flag bit along with the
PF_SPREAD_PAGE and PF_SPREAD_SLAB flag bits added earlier in this 'cpuset
memory spreading' patch set, one can check for the combination of any of these
special case memory placement mechanisms with a single test of the current
tasks task_struct flags.

This patch also tightens up the code, to save a few bytes of kernel text
space, and moves some of it out of line.  Due to the nested inlines called
from multiple places, we were ending up with three copies of this code, which
once we get off the main code path (for local node allocation) seems a bit
wasteful of instruction memory.

Signed-off-by: Paul Jackson <pj@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 28 insertions, 13 deletions

diff --git a/mm/slab.c b/mm/slab.c
index de516658d3d..f80b52388a1 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -899,6 +899,7 @@ static struct array_cache *alloc_arraycache(int node, int entries,
 
 #ifdef CONFIG_NUMA
 static void *__cache_alloc_node(struct kmem_cache *, gfp_t, int);
+static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
 
 static struct array_cache **alloc_alien_cache(int node, int limit)
 {
@@ -2808,19 +2809,11 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
         struct array_cache *ac;
 
 #ifdef CONFIG_NUMA
-        if (unlikely(current->mempolicy && !in_interrupt())) {
-                int nid = slab_node(current->mempolicy);
-
-                if (nid != numa_node_id())
-                        return __cache_alloc_node(cachep, flags, nid);
-        }
-        if (unlikely(cpuset_do_slab_mem_spread() &&
-                                        (cachep->flags & SLAB_MEM_SPREAD) &&
-                                        !in_interrupt())) {
-                int nid = cpuset_mem_spread_node();
-
-                if (nid != numa_node_id())
-                        return __cache_alloc_node(cachep, flags, nid);
+        if (unlikely(current->flags & (PF_SPREAD_PAGE | PF_SPREAD_SLAB |
+                                                        PF_MEMPOLICY))) {
+                objp = alternate_node_alloc(cachep, flags);
+                if (objp != NULL)
+                        return objp;
         }
 #endif
 
@@ -2856,6 +2849,28 @@ static __always_inline void *__cache_alloc(struct kmem_cache *cachep,
 
 #ifdef CONFIG_NUMA
 /*
+ * Try allocating on another node if PF_SPREAD_PAGE|PF_SPREAD_SLAB|PF_MEMPOLICY.
+ *
+ * If we are in_interrupt, then process context, including cpusets and
+ * mempolicy, may not apply and should not be used for allocation policy.
+ */
+static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
+{
+        int nid_alloc, nid_here;
+
+        if (in_interrupt())
+                return NULL;
+        nid_alloc = nid_here = numa_node_id();
+        if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
+                nid_alloc = cpuset_mem_spread_node();
+        else if (current->mempolicy)
+                nid_alloc = slab_node(current->mempolicy);
+        if (nid_alloc != nid_here)
+                return __cache_alloc_node(cachep, flags, nid_alloc);
+        return NULL;
+}
+
+/*
  * A interface to enable slab creation on nodeid
  */
 static void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,