sunrpc: keep a count of swapfiles associated with the rpc_clnt

Jerome reported seeing a warning pop when working with a swapfile on
NFS. The nfs_swap_activate can end up calling sk_set_memalloc while
holding the rcu_read_lock and that function can sleep.

To fix that, we need to take a reference to the xprt while holding the
rcu_read_lock, set the socket up for swapping and then drop that
reference. But, xprt_put is not exported and having NFS deal with the
underlying xprt is a bit of layering violation anyway.

Fix this by adding a set of activate/deactivate functions that take a
rpc_clnt pointer instead of an rpc_xprt, and have nfs_swap_activate and
nfs_swap_deactivate call those.

Also, add a per-rpc_clnt atomic counter to keep track of the number of
active swapfiles associated with it. When the counter does a 0->1
transition, we enable swapping on the xprt, when we do a 1->0 transition
we disable swapping on it.

This also allows us to be a bit more selective with the RPC_TASK_SWAPPER
flag. If non-swapper and swapper clnts are sharing a xprt, then we only
need to flag the tasks from the swapper clnt with that flag.

Acked-by: Mel Gorman <mgorman@suse.de>
Reported-by: Jerome Marchand <jmarchan@redhat.com>
Signed-off-by: Jeff Layton <jeff.layton@primarydata.com>
Reviewed-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>

1 files changed, 58 insertions, 9 deletions

diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index f6717170480e..0ba65156a62b 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -891,15 +891,8 @@ void rpc_task_set_client(struct rpc_task *task, struct rpc_clnt *clnt)
                         task->tk_flags |= RPC_TASK_SOFT;
                 if (clnt->cl_noretranstimeo)
                         task->tk_flags |= RPC_TASK_NO_RETRANS_TIMEOUT;
-                if (sk_memalloc_socks()) {
-                        struct rpc_xprt *xprt;
-
-                        rcu_read_lock();
-                        xprt = rcu_dereference(clnt->cl_xprt);
-                        if (xprt->swapper)
-                                task->tk_flags |= RPC_TASK_SWAPPER;
-                        rcu_read_unlock();
-                }
+                if (atomic_read(&clnt->cl_swapper))
+                        task->tk_flags |= RPC_TASK_SWAPPER;
                 /* Add to the client's list of all tasks */
                 spin_lock(&clnt->cl_lock);
                 list_add_tail(&task->tk_task, &clnt->cl_tasks);
@@ -2479,3 +2472,59 @@ void rpc_show_tasks(struct net *net)
         spin_unlock(&sn->rpc_client_lock);
 }
 #endif
+
+#if IS_ENABLED(CONFIG_SUNRPC_SWAP)
+int
+rpc_clnt_swap_activate(struct rpc_clnt *clnt)
+{
+        int ret = 0;
+        struct rpc_xprt *xprt;
+
+        if (atomic_inc_return(&clnt->cl_swapper) == 1) {
+retry:
+                rcu_read_lock();
+                xprt = xprt_get(rcu_dereference(clnt->cl_xprt));
+                rcu_read_unlock();
+                if (!xprt) {
+                        /*
+                         * If we didn't get a reference, then we likely are
+                         * racing with a migration event. Wait for a grace
+                         * period and try again.
+                         */
+                        synchronize_rcu();
+                        goto retry;
+                }
+
+                ret = xs_swapper(xprt, 1);
+                xprt_put(xprt);
+        }
+        return ret;
+}
+EXPORT_SYMBOL_GPL(rpc_clnt_swap_activate);
+
+void
+rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
+{
+        struct rpc_xprt *xprt;
+
+        if (atomic_dec_if_positive(&clnt->cl_swapper) == 0) {
+retry:
+                rcu_read_lock();
+                xprt = xprt_get(rcu_dereference(clnt->cl_xprt));
+                rcu_read_unlock();
+                if (!xprt) {
+                        /*
+                         * If we didn't get a reference, then we likely are
+                         * racing with a migration event. Wait for a grace
+                         * period and try again.
+                         */
+                        synchronize_rcu();
+                        goto retry;
+                }
+
+                xs_swapper(xprt, 0);
+                xprt_put(xprt);
+        }
+}
+EXPORT_SYMBOL_GPL(rpc_clnt_swap_deactivate);
+#endif /* CONFIG_SUNRPC_SWAP */