[PATCH] RPC: parametrize various transport connect timeouts

 Each transport implementation can now set unique bind, connect,
 reestablishment, and idle timeout values.  These are variables,
 allowing the values to be modified dynamically.  This permits
 exponential backoff of any of these values, for instance.

 As an example, we implement exponential backoff for the connection
 reestablishment timeout.

 Test-plan:
 Destructive testing (unplugging the network temporarily).  Connectathon
 with UDP and TCP.

 Signed-off-by: Chuck Lever <cel@netapp.com>
 Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

6 files changed, 84 insertions, 34 deletions

diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index b6a1ca508e60..062911e7ceb5 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -369,8 +369,8 @@ static void nfs_init_timeout_values(struct rpc_timeout *to, int proto, unsigned
         case IPPROTO_TCP:
                 if (!to->to_initval)
                         to->to_initval = 60 * HZ;
-                if (to->to_initval > RPC_MAX_TCP_TIMEOUT)
-                        to->to_initval = RPC_MAX_TCP_TIMEOUT;
+                if (to->to_initval > NFS_MAX_TCP_TIMEOUT)
+                        to->to_initval = NFS_MAX_TCP_TIMEOUT;
                 to->to_increment = to->to_initval;
                 to->to_maxval = to->to_initval + (to->to_increment * to->to_retries);
                 to->to_exponential = 0;
@@ -379,9 +379,9 @@ static void nfs_init_timeout_values(struct rpc_timeout *to, int proto, unsigned
         default:
                 if (!to->to_initval)
                         to->to_initval = 11 * HZ / 10;
-                if (to->to_initval > RPC_MAX_UDP_TIMEOUT)
-                        to->to_initval = RPC_MAX_UDP_TIMEOUT;
-                to->to_maxval = RPC_MAX_UDP_TIMEOUT;
+                if (to->to_initval > NFS_MAX_UDP_TIMEOUT)
+                        to->to_initval = NFS_MAX_UDP_TIMEOUT;
+                to->to_maxval = NFS_MAX_UDP_TIMEOUT;
                 to->to_exponential = 1;
                 break;
         }
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 9a6047ff1b25..7bac2785c6e4 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -41,6 +41,10 @@
 #define NFS_MAX_FILE_IO_BUFFER_SIZE     32768
 #define NFS_DEF_FILE_IO_BUFFER_SIZE     4096
 
+/* Default timeout values */
+#define NFS_MAX_UDP_TIMEOUT     (60*HZ)
+#define NFS_MAX_TCP_TIMEOUT     (600*HZ)
+
 /*
  * superblock magic number for NFS
  */
diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 9d9266cf8a36..2543adf18551 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -22,28 +22,6 @@ extern unsigned int xprt_tcp_slot_table_entries;
 #define RPC_DEF_SLOT_TABLE      (16U)
 #define RPC_MAX_SLOT_TABLE      (128U)
 
-/* Default timeout values */
-#define RPC_MAX_UDP_TIMEOUT     (60*HZ)
-#define RPC_MAX_TCP_TIMEOUT     (600*HZ)
-
-/*
- * Wait duration for an RPC TCP connection to be established.  Solaris
- * NFS over TCP uses 60 seconds, for example, which is in line with how
- * long a server takes to reboot.
- */
-#define RPC_CONNECT_TIMEOUT     (60*HZ)
-
-/*
- * Delay an arbitrary number of seconds before attempting to reconnect
- * after an error.
- */
-#define RPC_REESTABLISH_TIMEOUT (15*HZ)
-
-/*
- * RPC transport idle timeout.
- */
-#define RPC_IDLE_DISCONNECT_TIMEOUT     (5*60*HZ)
-
 /*
  * RPC call and reply header size as number of 32bit words (verifier
  * size computed separately)
@@ -182,14 +160,19 @@ struct rpc_xprt {
         /*
          * Connection of transports
          */
+        unsigned long           connect_timeout,
+                                bind_timeout,
+                                reestablish_timeout;
         struct work_struct      connect_worker;
         unsigned short          port;
+
         /*
          * Disconnection of idle transports
          */
         struct work_struct      task_cleanup;
         struct timer_list       timer;
-        unsigned long           last_used;
+        unsigned long           last_used,
+                                idle_timeout;
 
         /*
          * Send stuff
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index cc1b773a79d3..24b44e73f391 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -740,7 +740,7 @@ call_bind(struct rpc_task *task)
         task->tk_action = call_connect;
         if (!clnt->cl_port) {
                 task->tk_action = call_bind_status;
-                task->tk_timeout = RPC_CONNECT_TIMEOUT;
+                task->tk_timeout = task->tk_xprt->bind_timeout;
                 rpc_getport(task, clnt);
         }
 }
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index 0458319a1bdd..215be0d0ef6b 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -551,7 +551,7 @@ void xprt_connect(struct rpc_task *task)
                 if (task->tk_rqstp)
                         task->tk_rqstp->rq_bytes_sent = 0;
 
-                task->tk_timeout = RPC_CONNECT_TIMEOUT;
+                task->tk_timeout = xprt->connect_timeout;
                 rpc_sleep_on(&xprt->pending, task, xprt_connect_status, NULL);
                 xprt->ops->connect(task);
         }
@@ -763,7 +763,6 @@ void xprt_transmit(struct rpc_task *task)
 
         switch (status) {
         case -ECONNREFUSED:
-                task->tk_timeout = RPC_REESTABLISH_TIMEOUT;
                 rpc_sleep_on(&xprt->sending, task, NULL, NULL);
         case -EAGAIN:
         case -ENOTCONN:
@@ -857,7 +856,7 @@ void xprt_release(struct rpc_task *task)
         xprt->last_used = jiffies;
         if (list_empty(&xprt->recv) && !xprt->shutdown)
                 mod_timer(&xprt->timer,
-                                xprt->last_used + RPC_IDLE_DISCONNECT_TIMEOUT);
+                                xprt->last_used + xprt->idle_timeout);
         spin_unlock_bh(&xprt->transport_lock);
         task->tk_rqstp = NULL;
         memset(req, 0, sizeof(*req));   /* mark unused */
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 88ac71fcd335..06c2d95484e0 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -41,6 +41,50 @@
  */
 #define XS_SENDMSG_RETRY        (10U)
 
+/*
+ * Time out for an RPC UDP socket connect.  UDP socket connects are
+ * synchronous, but we set a timeout anyway in case of resource
+ * exhaustion on the local host.
+ */
+#define XS_UDP_CONN_TO          (5U * HZ)
+
+/*
+ * Wait duration for an RPC TCP connection to be established.  Solaris
+ * NFS over TCP uses 60 seconds, for example, which is in line with how
+ * long a server takes to reboot.
+ */
+#define XS_TCP_CONN_TO          (60U * HZ)
+
+/*
+ * Wait duration for a reply from the RPC portmapper.
+ */
+#define XS_BIND_TO              (60U * HZ)
+
+/*
+ * Delay if a UDP socket connect error occurs.  This is most likely some
+ * kind of resource problem on the local host.
+ */
+#define XS_UDP_REEST_TO         (2U * HZ)
+
+/*
+ * The reestablish timeout allows clients to delay for a bit before attempting
+ * to reconnect to a server that just dropped our connection.
+ *
+ * We implement an exponential backoff when trying to reestablish a TCP
+ * transport connection with the server.  Some servers like to drop a TCP
+ * connection when they are overworked, so we start with a short timeout and
+ * increase over time if the server is down or not responding.
+ */
+#define XS_TCP_INIT_REEST_TO    (3U * HZ)
+#define XS_TCP_MAX_REEST_TO     (5U * 60 * HZ)
+
+/*
+ * TCP idle timeout; client drops the transport socket if it is idle
+ * for this long.  Note that we also timeout UDP sockets to prevent
+ * holding port numbers when there is no RPC traffic.
+ */
+#define XS_IDLE_DISC_TO         (5U * 60 * HZ)
+
 #ifdef RPC_DEBUG
 # undef  RPC_DEBUG_DATA
 # define RPCDBG_FACILITY        RPCDBG_TRANS
@@ -739,6 +783,7 @@ static void xs_tcp_state_change(struct sock *sk)
                         xprt->tcp_reclen = 0;
                         xprt->tcp_copied = 0;
                         xprt->tcp_flags = XPRT_COPY_RECM | XPRT_COPY_XID;
+                        xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
                         xprt_wake_pending_tasks(xprt, 0);
                 }
                 spin_unlock_bh(&xprt->transport_lock);
@@ -1066,6 +1111,13 @@ out_clear:
  * @task: address of RPC task that manages state of connect request
  *
  * TCP: If the remote end dropped the connection, delay reconnecting.
+ *
+ * UDP socket connects are synchronous, but we use a work queue anyway
+ * to guarantee that even unprivileged user processes can set up a
+ * socket on a privileged port.
+ *
+ * If a UDP socket connect fails, the delay behavior here prevents
+ * retry floods (hard mounts).
  */
 static void xs_connect(struct rpc_task *task)
 {
@@ -1075,9 +1127,13 @@ static void xs_connect(struct rpc_task *task)
                 return;
 
         if (xprt->sock != NULL) {
-                dprintk("RPC:      xs_connect delayed xprt %p\n", xprt);
+                dprintk("RPC:      xs_connect delayed xprt %p for %lu seconds\n",
+                                xprt, xprt->reestablish_timeout / HZ);
                 schedule_delayed_work(&xprt->connect_worker,
-                                        RPC_REESTABLISH_TIMEOUT);
+                                        xprt->reestablish_timeout);
+                xprt->reestablish_timeout <<= 1;
+                if (xprt->reestablish_timeout > XS_TCP_MAX_REEST_TO)
+                        xprt->reestablish_timeout = XS_TCP_MAX_REEST_TO;
         } else {
                 dprintk("RPC:      xs_connect scheduled xprt %p\n", xprt);
                 schedule_work(&xprt->connect_worker);
@@ -1139,6 +1195,10 @@ int xs_setup_udp(struct rpc_xprt *xprt, struct rpc_timeout *to)
         xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
 
         INIT_WORK(&xprt->connect_worker, xs_udp_connect_worker, xprt);
+        xprt->bind_timeout = XS_BIND_TO;
+        xprt->connect_timeout = XS_UDP_CONN_TO;
+        xprt->reestablish_timeout = XS_UDP_REEST_TO;
+        xprt->idle_timeout = XS_IDLE_DISC_TO;
 
         xprt->ops = &xs_udp_ops;
 
@@ -1176,6 +1236,10 @@ int xs_setup_tcp(struct rpc_xprt *xprt, struct rpc_timeout *to)
         xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
 
         INIT_WORK(&xprt->connect_worker, xs_tcp_connect_worker, xprt);
+        xprt->bind_timeout = XS_BIND_TO;
+        xprt->connect_timeout = XS_TCP_CONN_TO;
+        xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
+        xprt->idle_timeout = XS_IDLE_DISC_TO;
 
         xprt->ops = &xs_tcp_ops;