sunrpc: use better NUMA affinities

Use NUMA aware allocations to reduce latencies and increase throughput.

sunrpc kthreads can use kthread_create_on_node() if pool_mode is
"percpu" or "pernode", and svc_prepare_thread()/svc_init_buffer() can
also take into account NUMA node affinity for memory allocations.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: "J. Bruce Fields" <bfields@fieldses.org>
CC: Neil Brown <neilb@suse.de>
CC: David Miller <davem@davemloft.net>
Reviewed-by: Greg Banks <gnb@fastmail.fm>
[bfields@redhat.com: fix up caller nfs41_callback_up]
Signed-off-by: J. Bruce Fields <bfields@redhat.com>

1 files changed, 24 insertions, 9 deletions

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 6a69a1131fb7..30d70abb4e2c 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -295,6 +295,18 @@ svc_pool_map_put(void)
 }
 
 
+static int svc_pool_map_get_node(unsigned int pidx)
+{
+        const struct svc_pool_map *m = &svc_pool_map;
+
+        if (m->count) {
+                if (m->mode == SVC_POOL_PERCPU)
+                        return cpu_to_node(m->pool_to[pidx]);
+                if (m->mode == SVC_POOL_PERNODE)
+                        return m->pool_to[pidx];
+        }
+        return NUMA_NO_NODE;
+}
 /*
  * Set the given thread's cpus_allowed mask so that it
  * will only run on cpus in the given pool.
@@ -499,7 +511,7 @@ EXPORT_SYMBOL_GPL(svc_destroy);
  * We allocate pages and place them in rq_argpages.
  */
 static int
-svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
+svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
 {
         unsigned int pages, arghi;
 
@@ -513,7 +525,7 @@ svc_init_buffer(struct svc_rqst *rqstp, unsigned int size)
         arghi = 0;
         BUG_ON(pages > RPCSVC_MAXPAGES);
         while (pages) {
-                struct page *p = alloc_page(GFP_KERNEL);
+                struct page *p = alloc_pages_node(node, GFP_KERNEL, 0);
                 if (!p)
                         break;
                 rqstp->rq_pages[arghi++] = p;
@@ -536,11 +548,11 @@ svc_release_buffer(struct svc_rqst *rqstp)
 }
 
 struct svc_rqst *
-svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool)
+svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
 {
         struct svc_rqst *rqstp;
 
-        rqstp = kzalloc(sizeof(*rqstp), GFP_KERNEL);
+        rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
         if (!rqstp)
                 goto out_enomem;
 
@@ -554,15 +566,15 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool)
         rqstp->rq_server = serv;
         rqstp->rq_pool = pool;
 
-        rqstp->rq_argp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
+        rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
         if (!rqstp->rq_argp)
                 goto out_thread;
 
-        rqstp->rq_resp = kmalloc(serv->sv_xdrsize, GFP_KERNEL);
+        rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
         if (!rqstp->rq_resp)
                 goto out_thread;
 
-        if (!svc_init_buffer(rqstp, serv->sv_max_mesg))
+        if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
                 goto out_thread;
 
         return rqstp;
@@ -647,6 +659,7 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
         struct svc_pool *chosen_pool;
         int error = 0;
         unsigned int state = serv->sv_nrthreads-1;
+        int node;
 
         if (pool == NULL) {
                 /* The -1 assumes caller has done a svc_get() */
@@ -662,14 +675,16 @@ svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
                 nrservs--;
                 chosen_pool = choose_pool(serv, pool, &state);
 
-                rqstp = svc_prepare_thread(serv, chosen_pool);
+                node = svc_pool_map_get_node(chosen_pool->sp_id);
+                rqstp = svc_prepare_thread(serv, chosen_pool, node);
                 if (IS_ERR(rqstp)) {
                         error = PTR_ERR(rqstp);
                         break;
                 }
 
                 __module_get(serv->sv_module);
-                task = kthread_create(serv->sv_function, rqstp, serv->sv_name);
+                task = kthread_create_on_node(serv->sv_function, rqstp,
+                                              node, serv->sv_name);
                 if (IS_ERR(task)) {
                         error = PTR_ERR(task);
                         module_put(serv->sv_module);