[PATCH] RPC: parametrize various transport connect timeouts

 Each transport implementation can now set unique bind, connect,
 reestablishment, and idle timeout values.  These are variables,
 allowing the values to be modified dynamically.  This permits
 exponential backoff of any of these values, for instance.

 As an example, we implement exponential backoff for the connection
 reestablishment timeout.

 Test-plan:
 Destructive testing (unplugging the network temporarily).  Connectathon
 with UDP and TCP.

 Signed-off-by: Chuck Lever <cel@netapp.com>
 Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

3 files changed, 69 insertions, 6 deletions

diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index cc1b773a79d3..24b44e73f391 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -740,7 +740,7 @@ call_bind(struct rpc_task *task)
         task->tk_action = call_connect;
         if (!clnt->cl_port) {
                 task->tk_action = call_bind_status;
-                task->tk_timeout = RPC_CONNECT_TIMEOUT;
+                task->tk_timeout = task->tk_xprt->bind_timeout;
                 rpc_getport(task, clnt);
         }
 }
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index 0458319a1bdd..215be0d0ef6b 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -551,7 +551,7 @@ void xprt_connect(struct rpc_task *task)
                 if (task->tk_rqstp)
                         task->tk_rqstp->rq_bytes_sent = 0;
 
-                task->tk_timeout = RPC_CONNECT_TIMEOUT;
+                task->tk_timeout = xprt->connect_timeout;
                 rpc_sleep_on(&xprt->pending, task, xprt_connect_status, NULL);
                 xprt->ops->connect(task);
         }
@@ -763,7 +763,6 @@ void xprt_transmit(struct rpc_task *task)
 
         switch (status) {
         case -ECONNREFUSED:
-                task->tk_timeout = RPC_REESTABLISH_TIMEOUT;
                 rpc_sleep_on(&xprt->sending, task, NULL, NULL);
         case -EAGAIN:
         case -ENOTCONN:
@@ -857,7 +856,7 @@ void xprt_release(struct rpc_task *task)
         xprt->last_used = jiffies;
         if (list_empty(&xprt->recv) && !xprt->shutdown)
                 mod_timer(&xprt->timer,
-                                xprt->last_used + RPC_IDLE_DISCONNECT_TIMEOUT);
+                                xprt->last_used + xprt->idle_timeout);
         spin_unlock_bh(&xprt->transport_lock);
         task->tk_rqstp = NULL;
         memset(req, 0, sizeof(*req));   /* mark unused */
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 88ac71fcd335..06c2d95484e0 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -41,6 +41,50 @@
  */
 #define XS_SENDMSG_RETRY        (10U)
 
+/*
+ * Time out for an RPC UDP socket connect.  UDP socket connects are
+ * synchronous, but we set a timeout anyway in case of resource
+ * exhaustion on the local host.
+ */
+#define XS_UDP_CONN_TO          (5U * HZ)
+
+/*
+ * Wait duration for an RPC TCP connection to be established.  Solaris
+ * NFS over TCP uses 60 seconds, for example, which is in line with how
+ * long a server takes to reboot.
+ */
+#define XS_TCP_CONN_TO          (60U * HZ)
+
+/*
+ * Wait duration for a reply from the RPC portmapper.
+ */
+#define XS_BIND_TO              (60U * HZ)
+
+/*
+ * Delay if a UDP socket connect error occurs.  This is most likely some
+ * kind of resource problem on the local host.
+ */
+#define XS_UDP_REEST_TO         (2U * HZ)
+
+/*
+ * The reestablish timeout allows clients to delay for a bit before attempting
+ * to reconnect to a server that just dropped our connection.
+ *
+ * We implement an exponential backoff when trying to reestablish a TCP
+ * transport connection with the server.  Some servers like to drop a TCP
+ * connection when they are overworked, so we start with a short timeout and
+ * increase over time if the server is down or not responding.
+ */
+#define XS_TCP_INIT_REEST_TO    (3U * HZ)
+#define XS_TCP_MAX_REEST_TO     (5U * 60 * HZ)
+
+/*
+ * TCP idle timeout; client drops the transport socket if it is idle
+ * for this long.  Note that we also timeout UDP sockets to prevent
+ * holding port numbers when there is no RPC traffic.
+ */
+#define XS_IDLE_DISC_TO         (5U * 60 * HZ)
+
 #ifdef RPC_DEBUG
 # undef  RPC_DEBUG_DATA
 # define RPCDBG_FACILITY        RPCDBG_TRANS
@@ -739,6 +783,7 @@ static void xs_tcp_state_change(struct sock *sk)
                         xprt->tcp_reclen = 0;
                         xprt->tcp_copied = 0;
                         xprt->tcp_flags = XPRT_COPY_RECM | XPRT_COPY_XID;
+                        xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
                         xprt_wake_pending_tasks(xprt, 0);
                 }
                 spin_unlock_bh(&xprt->transport_lock);
@@ -1066,6 +1111,13 @@ out_clear:
  * @task: address of RPC task that manages state of connect request
  *
  * TCP: If the remote end dropped the connection, delay reconnecting.
+ *
+ * UDP socket connects are synchronous, but we use a work queue anyway
+ * to guarantee that even unprivileged user processes can set up a
+ * socket on a privileged port.
+ *
+ * If a UDP socket connect fails, the delay behavior here prevents
+ * retry floods (hard mounts).
  */
 static void xs_connect(struct rpc_task *task)
 {
@@ -1075,9 +1127,13 @@ static void xs_connect(struct rpc_task *task)
                 return;
 
         if (xprt->sock != NULL) {
-                dprintk("RPC:      xs_connect delayed xprt %p\n", xprt);
+                dprintk("RPC:      xs_connect delayed xprt %p for %lu seconds\n",
+                                xprt, xprt->reestablish_timeout / HZ);
                 schedule_delayed_work(&xprt->connect_worker,
-                                        RPC_REESTABLISH_TIMEOUT);
+                                        xprt->reestablish_timeout);
+                xprt->reestablish_timeout <<= 1;
+                if (xprt->reestablish_timeout > XS_TCP_MAX_REEST_TO)
+                        xprt->reestablish_timeout = XS_TCP_MAX_REEST_TO;
         } else {
                 dprintk("RPC:      xs_connect scheduled xprt %p\n", xprt);
                 schedule_work(&xprt->connect_worker);
@@ -1139,6 +1195,10 @@ int xs_setup_udp(struct rpc_xprt *xprt, struct rpc_timeout *to)
         xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
 
         INIT_WORK(&xprt->connect_worker, xs_udp_connect_worker, xprt);
+        xprt->bind_timeout = XS_BIND_TO;
+        xprt->connect_timeout = XS_UDP_CONN_TO;
+        xprt->reestablish_timeout = XS_UDP_REEST_TO;
+        xprt->idle_timeout = XS_IDLE_DISC_TO;
 
         xprt->ops = &xs_udp_ops;
 
@@ -1176,6 +1236,10 @@ int xs_setup_tcp(struct rpc_xprt *xprt, struct rpc_timeout *to)
         xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
 
         INIT_WORK(&xprt->connect_worker, xs_tcp_connect_worker, xprt);
+        xprt->bind_timeout = XS_BIND_TO;
+        xprt->connect_timeout = XS_TCP_CONN_TO;
+        xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
+        xprt->idle_timeout = XS_IDLE_DISC_TO;
 
         xprt->ops = &xs_tcp_ops;