7 files changed, 635 insertions, 44 deletions

diff --git a/fs/ocfs2/cluster/Makefile b/fs/ocfs2/cluster/Makefile
index cdd162f13650..bc8c5e7d8608 100644
--- a/fs/ocfs2/cluster/Makefile
+++ b/fs/ocfs2/cluster/Makefile
@@ -1,4 +1,4 @@
 obj-$(CONFIG_OCFS2_FS) += ocfs2_nodemanager.o
 
 ocfs2_nodemanager-objs := heartbeat.o masklog.o sys.o nodemanager.o \
-        quorum.o tcp.o ver.o
+        quorum.o tcp.o netdebug.o ver.o
diff --git a/fs/ocfs2/cluster/netdebug.c b/fs/ocfs2/cluster/netdebug.c
new file mode 100644
index 000000000000..7bf3c0ea7bd9
--- /dev/null
+++ b/fs/ocfs2/cluster/netdebug.c
@@ -0,0 +1,441 @@
+/* -*- mode: c; c-basic-offset: 8; -*-
+ * vim: noexpandtab sw=8 ts=8 sts=0:
+ *
+ * netdebug.c
+ *
+ * debug functionality for o2net
+ *
+ * Copyright (C) 2005, 2008 Oracle.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ *
+ */
+
+#ifdef CONFIG_DEBUG_FS
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/idr.h>
+#include <linux/kref.h>
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+
+#include <linux/uaccess.h>
+
+#include "tcp.h"
+#include "nodemanager.h"
+#define MLOG_MASK_PREFIX ML_TCP
+#include "masklog.h"
+
+#include "tcp_internal.h"
+
+#define O2NET_DEBUG_DIR         "o2net"
+#define SC_DEBUG_NAME           "sock_containers"
+#define NST_DEBUG_NAME          "send_tracking"
+
+static struct dentry *o2net_dentry;
+static struct dentry *sc_dentry;
+static struct dentry *nst_dentry;
+
+static DEFINE_SPINLOCK(o2net_debug_lock);
+
+static LIST_HEAD(sock_containers);
+static LIST_HEAD(send_tracking);
+
+void o2net_debug_add_nst(struct o2net_send_tracking *nst)
+{
+        spin_lock(&o2net_debug_lock);
+        list_add(&nst->st_net_debug_item, &send_tracking);
+        spin_unlock(&o2net_debug_lock);
+}
+
+void o2net_debug_del_nst(struct o2net_send_tracking *nst)
+{
+        spin_lock(&o2net_debug_lock);
+        if (!list_empty(&nst->st_net_debug_item))
+                list_del_init(&nst->st_net_debug_item);
+        spin_unlock(&o2net_debug_lock);
+}
+
+static struct o2net_send_tracking
+                        *next_nst(struct o2net_send_tracking *nst_start)
+{
+        struct o2net_send_tracking *nst, *ret = NULL;
+
+        assert_spin_locked(&o2net_debug_lock);
+
+        list_for_each_entry(nst, &nst_start->st_net_debug_item,
+                            st_net_debug_item) {
+                /* discover the head of the list */
+                if (&nst->st_net_debug_item == &send_tracking)
+                        break;
+
+                /* use st_task to detect real nsts in the list */
+                if (nst->st_task != NULL) {
+                        ret = nst;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+static void *nst_seq_start(struct seq_file *seq, loff_t *pos)
+{
+        struct o2net_send_tracking *nst, *dummy_nst = seq->private;
+
+        spin_lock(&o2net_debug_lock);
+        nst = next_nst(dummy_nst);
+        spin_unlock(&o2net_debug_lock);
+
+        return nst;
+}
+
+static void *nst_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+        struct o2net_send_tracking *nst, *dummy_nst = seq->private;
+
+        spin_lock(&o2net_debug_lock);
+        nst = next_nst(dummy_nst);
+        list_del_init(&dummy_nst->st_net_debug_item);
+        if (nst)
+                list_add(&dummy_nst->st_net_debug_item,
+                         &nst->st_net_debug_item);
+        spin_unlock(&o2net_debug_lock);
+
+        return nst; /* unused, just needs to be null when done */
+}
+
+static int nst_seq_show(struct seq_file *seq, void *v)
+{
+        struct o2net_send_tracking *nst, *dummy_nst = seq->private;
+
+        spin_lock(&o2net_debug_lock);
+        nst = next_nst(dummy_nst);
+
+        if (nst != NULL) {
+                /* get_task_comm isn't exported.  oh well. */
+                seq_printf(seq, "%p:\n"
+                           "  pid:          %lu\n"
+                           "  tgid:         %lu\n"
+                           "  process name: %s\n"
+                           "  node:         %u\n"
+                           "  sc:           %p\n"
+                           "  message id:   %d\n"
+                           "  message type: %u\n"
+                           "  message key:  0x%08x\n"
+                           "  sock acquiry: %lu.%lu\n"
+                           "  send start:   %lu.%lu\n"
+                           "  wait start:   %lu.%lu\n",
+                           nst, (unsigned long)nst->st_task->pid,
+                           (unsigned long)nst->st_task->tgid,
+                           nst->st_task->comm, nst->st_node,
+                           nst->st_sc, nst->st_id, nst->st_msg_type,
+                           nst->st_msg_key,
+                           nst->st_sock_time.tv_sec, nst->st_sock_time.tv_usec,
+                           nst->st_send_time.tv_sec, nst->st_send_time.tv_usec,
+                           nst->st_status_time.tv_sec,
+                           nst->st_status_time.tv_usec);
+        }
+
+        spin_unlock(&o2net_debug_lock);
+
+        return 0;
+}
+
+static void nst_seq_stop(struct seq_file *seq, void *v)
+{
+}
+
+static struct seq_operations nst_seq_ops = {
+        .start = nst_seq_start,
+        .next = nst_seq_next,
+        .stop = nst_seq_stop,
+        .show = nst_seq_show,
+};
+
+static int nst_fop_open(struct inode *inode, struct file *file)
+{
+        struct o2net_send_tracking *dummy_nst;
+        struct seq_file *seq;
+        int ret;
+
+        dummy_nst = kmalloc(sizeof(struct o2net_send_tracking), GFP_KERNEL);
+        if (dummy_nst == NULL) {
+                ret = -ENOMEM;
+                goto out;
+        }
+        dummy_nst->st_task = NULL;
+
+        ret = seq_open(file, &nst_seq_ops);
+        if (ret)
+                goto out;
+
+        seq = file->private_data;
+        seq->private = dummy_nst;
+        o2net_debug_add_nst(dummy_nst);
+
+        dummy_nst = NULL;
+
+out:
+        kfree(dummy_nst);
+        return ret;
+}
+
+static int nst_fop_release(struct inode *inode, struct file *file)
+{
+        struct seq_file *seq = file->private_data;
+        struct o2net_send_tracking *dummy_nst = seq->private;
+
+        o2net_debug_del_nst(dummy_nst);
+        return seq_release_private(inode, file);
+}
+
+static struct file_operations nst_seq_fops = {
+        .open = nst_fop_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = nst_fop_release,
+};
+
+void o2net_debug_add_sc(struct o2net_sock_container *sc)
+{
+        spin_lock(&o2net_debug_lock);
+        list_add(&sc->sc_net_debug_item, &sock_containers);
+        spin_unlock(&o2net_debug_lock);
+}
+
+void o2net_debug_del_sc(struct o2net_sock_container *sc)
+{
+        spin_lock(&o2net_debug_lock);
+        list_del_init(&sc->sc_net_debug_item);
+        spin_unlock(&o2net_debug_lock);
+}
+
+static struct o2net_sock_container
+                        *next_sc(struct o2net_sock_container *sc_start)
+{
+        struct o2net_sock_container *sc, *ret = NULL;
+
+        assert_spin_locked(&o2net_debug_lock);
+
+        list_for_each_entry(sc, &sc_start->sc_net_debug_item,
+                            sc_net_debug_item) {
+                /* discover the head of the list miscast as a sc */
+                if (&sc->sc_net_debug_item == &sock_containers)
+                        break;
+
+                /* use sc_page to detect real scs in the list */
+                if (sc->sc_page != NULL) {
+                        ret = sc;
+                        break;
+                }
+        }
+
+        return ret;
+}
+
+static void *sc_seq_start(struct seq_file *seq, loff_t *pos)
+{
+        struct o2net_sock_container *sc, *dummy_sc = seq->private;
+
+        spin_lock(&o2net_debug_lock);
+        sc = next_sc(dummy_sc);
+        spin_unlock(&o2net_debug_lock);
+
+        return sc;
+}
+
+static void *sc_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+        struct o2net_sock_container *sc, *dummy_sc = seq->private;
+
+        spin_lock(&o2net_debug_lock);
+        sc = next_sc(dummy_sc);
+        list_del_init(&dummy_sc->sc_net_debug_item);
+        if (sc)
+                list_add(&dummy_sc->sc_net_debug_item, &sc->sc_net_debug_item);
+        spin_unlock(&o2net_debug_lock);
+
+        return sc; /* unused, just needs to be null when done */
+}
+
+#define TV_SEC_USEC(TV) TV.tv_sec, TV.tv_usec
+
+static int sc_seq_show(struct seq_file *seq, void *v)
+{
+        struct o2net_sock_container *sc, *dummy_sc = seq->private;
+
+        spin_lock(&o2net_debug_lock);
+        sc = next_sc(dummy_sc);
+
+        if (sc != NULL) {
+                struct inet_sock *inet = NULL;
+
+                __be32 saddr = 0, daddr = 0;
+                __be16 sport = 0, dport = 0;
+
+                if (sc->sc_sock) {
+                        inet = inet_sk(sc->sc_sock->sk);
+                        /* the stack's structs aren't sparse endian clean */
+                        saddr = (__force __be32)inet->saddr;
+                        daddr = (__force __be32)inet->daddr;
+                        sport = (__force __be16)inet->sport;
+                        dport = (__force __be16)inet->dport;
+                }
+
+                /* XXX sigh, inet-> doesn't have sparse annotation so any
+                 * use of it here generates a warning with -Wbitwise */
+                seq_printf(seq, "%p:\n"
+                           "  krefs:           %d\n"
+                           "  sock:            %u.%u.%u.%u:%u -> "
+                                              "%u.%u.%u.%u:%u\n"
+                           "  remote node:     %s\n"
+                           "  page off:        %zu\n"
+                           "  handshake ok:    %u\n"
+                           "  timer:           %lu.%lu\n"
+                           "  data ready:      %lu.%lu\n"
+                           "  advance start:   %lu.%lu\n"
+                           "  advance stop:    %lu.%lu\n"
+                           "  func start:      %lu.%lu\n"
+                           "  func stop:       %lu.%lu\n"
+                           "  func key:        %u\n"
+                           "  func type:       %u\n",
+                           sc,
+                           atomic_read(&sc->sc_kref.refcount),
+                           NIPQUAD(saddr), inet ? ntohs(sport) : 0,
+                           NIPQUAD(daddr), inet ? ntohs(dport) : 0,
+                           sc->sc_node->nd_name,
+                           sc->sc_page_off,
+                           sc->sc_handshake_ok,
+                           TV_SEC_USEC(sc->sc_tv_timer),
+                           TV_SEC_USEC(sc->sc_tv_data_ready),
+                           TV_SEC_USEC(sc->sc_tv_advance_start),
+                           TV_SEC_USEC(sc->sc_tv_advance_stop),
+                           TV_SEC_USEC(sc->sc_tv_func_start),
+                           TV_SEC_USEC(sc->sc_tv_func_stop),
+                           sc->sc_msg_key,
+                           sc->sc_msg_type);
+        }
+
+
+        spin_unlock(&o2net_debug_lock);
+
+        return 0;
+}
+
+static void sc_seq_stop(struct seq_file *seq, void *v)
+{
+}
+
+static struct seq_operations sc_seq_ops = {
+        .start = sc_seq_start,
+        .next = sc_seq_next,
+        .stop = sc_seq_stop,
+        .show = sc_seq_show,
+};
+
+static int sc_fop_open(struct inode *inode, struct file *file)
+{
+        struct o2net_sock_container *dummy_sc;
+        struct seq_file *seq;
+        int ret;
+
+        dummy_sc = kmalloc(sizeof(struct o2net_sock_container), GFP_KERNEL);
+        if (dummy_sc == NULL) {
+                ret = -ENOMEM;
+                goto out;
+        }
+        dummy_sc->sc_page = NULL;
+
+        ret = seq_open(file, &sc_seq_ops);
+        if (ret)
+                goto out;
+
+        seq = file->private_data;
+        seq->private = dummy_sc;
+        o2net_debug_add_sc(dummy_sc);
+
+        dummy_sc = NULL;
+
+out:
+        kfree(dummy_sc);
+        return ret;
+}
+
+static int sc_fop_release(struct inode *inode, struct file *file)
+{
+        struct seq_file *seq = file->private_data;
+        struct o2net_sock_container *dummy_sc = seq->private;
+
+        o2net_debug_del_sc(dummy_sc);
+        return seq_release_private(inode, file);
+}
+
+static struct file_operations sc_seq_fops = {
+        .open = sc_fop_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = sc_fop_release,
+};
+
+int o2net_debugfs_init(void)
+{
+        o2net_dentry = debugfs_create_dir(O2NET_DEBUG_DIR, NULL);
+        if (!o2net_dentry) {
+                mlog_errno(-ENOMEM);
+                goto bail;
+        }
+
+        nst_dentry = debugfs_create_file(NST_DEBUG_NAME, S_IFREG|S_IRUSR,
+                                         o2net_dentry, NULL,
+                                         &nst_seq_fops);
+        if (!nst_dentry) {
+                mlog_errno(-ENOMEM);
+                goto bail;
+        }
+
+        sc_dentry = debugfs_create_file(SC_DEBUG_NAME, S_IFREG|S_IRUSR,
+                                        o2net_dentry, NULL,
+                                        &sc_seq_fops);
+        if (!sc_dentry) {
+                mlog_errno(-ENOMEM);
+                goto bail;
+        }
+
+        return 0;
+bail:
+        if (sc_dentry)
+                debugfs_remove(sc_dentry);
+        if (nst_dentry)
+                debugfs_remove(nst_dentry);
+        if (o2net_dentry)
+                debugfs_remove(o2net_dentry);
+        return -ENOMEM;
+}
+
+void o2net_debugfs_exit(void)
+{
+        if (sc_dentry)
+                debugfs_remove(sc_dentry);
+        if (nst_dentry)
+                debugfs_remove(nst_dentry);
+        if (o2net_dentry)
+                debugfs_remove(o2net_dentry);
+}
+
+#endif  /* CONFIG_DEBUG_FS */
diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c
index 709fba25bf7e..cf9401e8cd0b 100644
--- a/fs/ocfs2/cluster/nodemanager.c
+++ b/fs/ocfs2/cluster/nodemanager.c
@@ -959,7 +959,10 @@ static int __init init_o2nm(void)
         cluster_print_version();
 
         o2hb_init();
-        o2net_init();
+
+        ret = o2net_init();
+        if (ret)
+                goto out;
 
         ocfs2_table_header = register_sysctl_table(ocfs2_root_table);
         if (!ocfs2_table_header) {
diff --git a/fs/ocfs2/cluster/sys.c b/fs/ocfs2/cluster/sys.c
index 0c095ce7723d..98429fd68499 100644
--- a/fs/ocfs2/cluster/sys.c
+++ b/fs/ocfs2/cluster/sys.c
@@ -57,6 +57,7 @@ static struct kset *o2cb_kset;
 void o2cb_sys_shutdown(void)
 {
         mlog_sys_shutdown();
+        sysfs_remove_link(NULL, "o2cb");
         kset_unregister(o2cb_kset);
 }
 
@@ -68,6 +69,14 @@ int o2cb_sys_init(void)
         if (!o2cb_kset)
                 return -ENOMEM;
 
+        /*
+         * Create this symlink for backwards compatibility with old
+         * versions of ocfs2-tools which look for things in /sys/o2cb.
+         */
+        ret = sysfs_create_link(NULL, &o2cb_kset->kobj, "o2cb");
+        if (ret)
+                goto error;
+
         ret = sysfs_create_group(&o2cb_kset->kobj, &o2cb_attr_group);
         if (ret)
                 goto error;
diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c
index b8057c51b205..1e44ad14881a 100644
--- a/fs/ocfs2/cluster/tcp.c
+++ b/fs/ocfs2/cluster/tcp.c
@@ -142,23 +142,65 @@ static void o2net_idle_timer(unsigned long data);
 static void o2net_sc_postpone_idle(struct o2net_sock_container *sc);
 static void o2net_sc_reset_idle_timer(struct o2net_sock_container *sc);
 
-/*
- * FIXME: These should use to_o2nm_cluster_from_node(), but we end up
- * losing our parent link to the cluster during shutdown. This can be
- * solved by adding a pre-removal callback to configfs, or passing
- * around the cluster with the node. -jeffm
- */
-static inline int o2net_reconnect_delay(struct o2nm_node *node)
+static void o2net_init_nst(struct o2net_send_tracking *nst, u32 msgtype,
+                           u32 msgkey, struct task_struct *task, u8 node)
+{
+#ifdef CONFIG_DEBUG_FS
+        INIT_LIST_HEAD(&nst->st_net_debug_item);
+        nst->st_task = task;
+        nst->st_msg_type = msgtype;
+        nst->st_msg_key = msgkey;
+        nst->st_node = node;
+#endif
+}
+
+static void o2net_set_nst_sock_time(struct o2net_send_tracking *nst)
+{
+#ifdef CONFIG_DEBUG_FS
+        do_gettimeofday(&nst->st_sock_time);
+#endif
+}
+
+static void o2net_set_nst_send_time(struct o2net_send_tracking *nst)
+{
+#ifdef CONFIG_DEBUG_FS
+        do_gettimeofday(&nst->st_send_time);
+#endif
+}
+
+static void o2net_set_nst_status_time(struct o2net_send_tracking *nst)
+{
+#ifdef CONFIG_DEBUG_FS
+        do_gettimeofday(&nst->st_status_time);
+#endif
+}
+
+static void o2net_set_nst_sock_container(struct o2net_send_tracking *nst,
+                                         struct o2net_sock_container *sc)
+{
+#ifdef CONFIG_DEBUG_FS
+        nst->st_sc = sc;
+#endif
+}
+
+static void o2net_set_nst_msg_id(struct o2net_send_tracking *nst, u32 msg_id)
+{
+#ifdef CONFIG_DEBUG_FS
+        nst->st_id = msg_id;
+#endif
+}
+
+static inline int o2net_reconnect_delay(void)
 {
         return o2nm_single_cluster->cl_reconnect_delay_ms;
 }
 
-static inline int o2net_keepalive_delay(struct o2nm_node *node)
+static inline int o2net_keepalive_delay(void)
 {
         return o2nm_single_cluster->cl_keepalive_delay_ms;
 }
 
-static inline int o2net_idle_timeout(struct o2nm_node *node)
+static inline int o2net_idle_timeout(void)
 {
         return o2nm_single_cluster->cl_idle_timeout_ms;
 }
@@ -296,6 +338,7 @@ static void sc_kref_release(struct kref *kref)
         o2nm_node_put(sc->sc_node);
         sc->sc_node = NULL;
 
+        o2net_debug_del_sc(sc);
         kfree(sc);
 }
 
@@ -336,6 +379,7 @@ static struct o2net_sock_container *sc_alloc(struct o2nm_node *node)
 
         ret = sc;
         sc->sc_page = page;
+        o2net_debug_add_sc(sc);
         sc = NULL;
         page = NULL;
 
@@ -399,8 +443,6 @@ static void o2net_set_nn_state(struct o2net_node *nn,
         mlog_bug_on_msg(err && valid, "err %d valid %u\n", err, valid);
         mlog_bug_on_msg(valid && !sc, "valid %u sc %p\n", valid, sc);
 
-        /* we won't reconnect after our valid conn goes away for
-         * this hb iteration.. here so it shows up in the logs */
         if (was_valid && !valid && err == 0)
                 err = -ENOTCONN;
 
@@ -430,11 +472,6 @@ static void o2net_set_nn_state(struct o2net_node *nn,
 
         if (!was_valid && valid) {
                 o2quo_conn_up(o2net_num_from_nn(nn));
-                /* this is a bit of a hack.  we only try reconnecting
-                 * when heartbeating starts until we get a connection.
-                 * if that connection then dies we don't try reconnecting.
-                 * the only way to start connecting again is to down
-                 * heartbeat and bring it back up. */
                 cancel_delayed_work(&nn->nn_connect_expired);
                 printk(KERN_INFO "o2net: %s " SC_NODEF_FMT "\n",
                        o2nm_this_node() > sc->sc_node->nd_num ?
@@ -451,12 +488,24 @@ static void o2net_set_nn_state(struct o2net_node *nn,
                 /* delay if we're withing a RECONNECT_DELAY of the
                  * last attempt */
                 delay = (nn->nn_last_connect_attempt +
-                         msecs_to_jiffies(o2net_reconnect_delay(NULL)))
+                         msecs_to_jiffies(o2net_reconnect_delay()))
                         - jiffies;
-                if (delay > msecs_to_jiffies(o2net_reconnect_delay(NULL)))
+                if (delay > msecs_to_jiffies(o2net_reconnect_delay()))
                         delay = 0;
                 mlog(ML_CONN, "queueing conn attempt in %lu jiffies\n", delay);
                 queue_delayed_work(o2net_wq, &nn->nn_connect_work, delay);
+
+                /*
+                 * Delay the expired work after idle timeout.
+                 *
+                 * We might have lots of failed connection attempts that run
+                 * through here but we only cancel the connect_expired work when
+                 * a connection attempt succeeds.  So only the first enqueue of
+                 * the connect_expired work will do anything.  The rest will see
+                 * that it's already queued and do nothing.
+                 */
+                delay += msecs_to_jiffies(o2net_idle_timeout());
+                queue_delayed_work(o2net_wq, &nn->nn_connect_expired, delay);
         }
 
         /* keep track of the nn's sc ref for the caller */
@@ -914,6 +963,9 @@ int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec,
         struct o2net_status_wait nsw = {
                 .ns_node_item = LIST_HEAD_INIT(nsw.ns_node_item),
         };
+        struct o2net_send_tracking nst;
+
+        o2net_init_nst(&nst, msg_type, key, current, target_node);
 
         if (o2net_wq == NULL) {
                 mlog(0, "attempt to tx without o2netd running\n");
@@ -939,6 +991,10 @@ int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec,
                 goto out;
         }
 
+        o2net_debug_add_nst(&nst);
+
+        o2net_set_nst_sock_time(&nst);
+
         ret = wait_event_interruptible(nn->nn_sc_wq,
                                        o2net_tx_can_proceed(nn, &sc, &error));
         if (!ret && error)
@@ -946,6 +1002,8 @@ int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec,
         if (ret)
                 goto out;
 
+        o2net_set_nst_sock_container(&nst, sc);
+
         veclen = caller_veclen + 1;
         vec = kmalloc(sizeof(struct kvec) * veclen, GFP_ATOMIC);
         if (vec == NULL) {
@@ -972,6 +1030,9 @@ int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec,
                 goto out;
 
         msg->msg_num = cpu_to_be32(nsw.ns_id);
+        o2net_set_nst_msg_id(&nst, nsw.ns_id);
+
+        o2net_set_nst_send_time(&nst);
 
         /* finally, convert the message header to network byte-order
          * and send */
@@ -986,6 +1047,7 @@ int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec,
         }
 
         /* wait on other node's handler */
+        o2net_set_nst_status_time(&nst);
         wait_event(nsw.ns_wq, o2net_nsw_completed(nn, &nsw));
 
         /* Note that we avoid overwriting the callers status return
@@ -998,6 +1060,7 @@ int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec,
         mlog(0, "woken, returning system status %d, user status %d\n",
              ret, nsw.ns_status);
 out:
+        o2net_debug_del_nst(&nst); /* must be before dropping sc and node */
         if (sc)
                 sc_put(sc);
         if (vec)
@@ -1154,23 +1217,23 @@ static int o2net_check_handshake(struct o2net_sock_container *sc)
          * but isn't. This can ultimately cause corruption.
          */
         if (be32_to_cpu(hand->o2net_idle_timeout_ms) !=
-                                o2net_idle_timeout(sc->sc_node)) {
+                                o2net_idle_timeout()) {
                 mlog(ML_NOTICE, SC_NODEF_FMT " uses a network idle timeout of "
                      "%u ms, but we use %u ms locally.  disconnecting\n",
                      SC_NODEF_ARGS(sc),
                      be32_to_cpu(hand->o2net_idle_timeout_ms),
-                     o2net_idle_timeout(sc->sc_node));
+                     o2net_idle_timeout());
                 o2net_ensure_shutdown(nn, sc, -ENOTCONN);
                 return -1;
         }
 
         if (be32_to_cpu(hand->o2net_keepalive_delay_ms) !=
-                        o2net_keepalive_delay(sc->sc_node)) {
+                        o2net_keepalive_delay()) {
                 mlog(ML_NOTICE, SC_NODEF_FMT " uses a keepalive delay of "
                      "%u ms, but we use %u ms locally.  disconnecting\n",
                      SC_NODEF_ARGS(sc),
                      be32_to_cpu(hand->o2net_keepalive_delay_ms),
-                     o2net_keepalive_delay(sc->sc_node));
+                     o2net_keepalive_delay());
                 o2net_ensure_shutdown(nn, sc, -ENOTCONN);
                 return -1;
         }
@@ -1193,6 +1256,7 @@ static int o2net_check_handshake(struct o2net_sock_container *sc)
          * shut down already */
         if (nn->nn_sc == sc) {
                 o2net_sc_reset_idle_timer(sc);
+                atomic_set(&nn->nn_timeout, 0);
                 o2net_set_nn_state(nn, sc, 1, 0);
         }
         spin_unlock(&nn->nn_lock);
@@ -1347,12 +1411,11 @@ static void o2net_initialize_handshake(void)
 {
         o2net_hand->o2hb_heartbeat_timeout_ms = cpu_to_be32(
                 O2HB_MAX_WRITE_TIMEOUT_MS);
-        o2net_hand->o2net_idle_timeout_ms = cpu_to_be32(
-                o2net_idle_timeout(NULL));
+        o2net_hand->o2net_idle_timeout_ms = cpu_to_be32(o2net_idle_timeout());
         o2net_hand->o2net_keepalive_delay_ms = cpu_to_be32(
-                o2net_keepalive_delay(NULL));
+                o2net_keepalive_delay());
         o2net_hand->o2net_reconnect_delay_ms = cpu_to_be32(
-                o2net_reconnect_delay(NULL));
+                o2net_reconnect_delay());
 }
 
 /* ------------------------------------------------------------ */
@@ -1391,14 +1454,15 @@ static void o2net_sc_send_keep_req(struct work_struct *work)
 static void o2net_idle_timer(unsigned long data)
 {
         struct o2net_sock_container *sc = (struct o2net_sock_container *)data;
+        struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num);
         struct timeval now;
 
         do_gettimeofday(&now);
 
         printk(KERN_INFO "o2net: connection to " SC_NODEF_FMT " has been idle for %u.%u "
              "seconds, shutting it down.\n", SC_NODEF_ARGS(sc),
-                     o2net_idle_timeout(sc->sc_node) / 1000,
-                     o2net_idle_timeout(sc->sc_node) % 1000);
+                     o2net_idle_timeout() / 1000,
+                     o2net_idle_timeout() % 1000);
         mlog(ML_NOTICE, "here are some times that might help debug the "
              "situation: (tmr %ld.%ld now %ld.%ld dr %ld.%ld adv "
              "%ld.%ld:%ld.%ld func (%08x:%u) %ld.%ld:%ld.%ld)\n",
@@ -1413,6 +1477,12 @@ static void o2net_idle_timer(unsigned long data)
              sc->sc_tv_func_start.tv_sec, (long) sc->sc_tv_func_start.tv_usec,
              sc->sc_tv_func_stop.tv_sec, (long) sc->sc_tv_func_stop.tv_usec);
 
+        /*
+         * Initialize the nn_timeout so that the next connection attempt
+         * will continue in o2net_start_connect.
+         */
+        atomic_set(&nn->nn_timeout, 1);
+
         o2net_sc_queue_work(sc, &sc->sc_shutdown_work);
 }
 
@@ -1420,10 +1490,10 @@ static void o2net_sc_reset_idle_timer(struct o2net_sock_container *sc)
 {
         o2net_sc_cancel_delayed_work(sc, &sc->sc_keepalive_work);
         o2net_sc_queue_delayed_work(sc, &sc->sc_keepalive_work,
-                      msecs_to_jiffies(o2net_keepalive_delay(sc->sc_node)));
+                      msecs_to_jiffies(o2net_keepalive_delay()));
         do_gettimeofday(&sc->sc_tv_timer);
         mod_timer(&sc->sc_idle_timeout,
-               jiffies + msecs_to_jiffies(o2net_idle_timeout(sc->sc_node)));
+               jiffies + msecs_to_jiffies(o2net_idle_timeout()));
 }
 
 static void o2net_sc_postpone_idle(struct o2net_sock_container *sc)
@@ -1447,6 +1517,7 @@ static void o2net_start_connect(struct work_struct *work)
         struct socket *sock = NULL;
         struct sockaddr_in myaddr = {0, }, remoteaddr = {0, };
         int ret = 0, stop;
+        unsigned int timeout;
 
         /* if we're greater we initiate tx, otherwise we accept */
         if (o2nm_this_node() <= o2net_num_from_nn(nn))
@@ -1466,8 +1537,17 @@ static void o2net_start_connect(struct work_struct *work)
         }
 
         spin_lock(&nn->nn_lock);
-        /* see if we already have one pending or have given up */
-        stop = (nn->nn_sc || nn->nn_persistent_error);
+        /*
+         * see if we already have one pending or have given up.
+         * For nn_timeout, it is set when we close the connection
+         * because of the idle time out. So it means that we have
+         * at least connected to that node successfully once,
+         * now try to connect to it again.
+         */
+        timeout = atomic_read(&nn->nn_timeout);
+        stop = (nn->nn_sc ||
+                (nn->nn_persistent_error &&
+                (nn->nn_persistent_error != -ENOTCONN || timeout == 0)));
         spin_unlock(&nn->nn_lock);
         if (stop)
                 goto out;
@@ -1555,8 +1635,8 @@ static void o2net_connect_expired(struct work_struct *work)
                 mlog(ML_ERROR, "no connection established with node %u after "
                      "%u.%u seconds, giving up and returning errors.\n",
                      o2net_num_from_nn(nn),
-                     o2net_idle_timeout(NULL) / 1000,
-                     o2net_idle_timeout(NULL) % 1000);
+                     o2net_idle_timeout() / 1000,
+                     o2net_idle_timeout() % 1000);
 
                 o2net_set_nn_state(nn, NULL, 0, -ENOTCONN);
         }
@@ -1579,6 +1659,7 @@ void o2net_disconnect_node(struct o2nm_node *node)
 
         /* don't reconnect until it's heartbeating again */
         spin_lock(&nn->nn_lock);
+        atomic_set(&nn->nn_timeout, 0);
         o2net_set_nn_state(nn, NULL, 0, -ENOTCONN);
         spin_unlock(&nn->nn_lock);
 
@@ -1610,20 +1691,15 @@ static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num,
 
         /* ensure an immediate connect attempt */
         nn->nn_last_connect_attempt = jiffies -
-                (msecs_to_jiffies(o2net_reconnect_delay(node)) + 1);
+                (msecs_to_jiffies(o2net_reconnect_delay()) + 1);
 
         if (node_num != o2nm_this_node()) {
-                /* heartbeat doesn't work unless a local node number is
-                 * configured and doing so brings up the o2net_wq, so we can
-                 * use it.. */
-                queue_delayed_work(o2net_wq, &nn->nn_connect_expired,
-                                   msecs_to_jiffies(o2net_idle_timeout(node)));
-
                 /* believe it or not, accept and node hearbeating testing
                  * can succeed for this node before we got here.. so
                  * only use set_nn_state to clear the persistent error
                  * if that hasn't already happened */
                 spin_lock(&nn->nn_lock);
+                atomic_set(&nn->nn_timeout, 0);
                 if (nn->nn_persistent_error)
                         o2net_set_nn_state(nn, NULL, 0, 0);
                 spin_unlock(&nn->nn_lock);
@@ -1747,6 +1823,7 @@ static int o2net_accept_one(struct socket *sock)
         new_sock = NULL;
 
         spin_lock(&nn->nn_lock);
+        atomic_set(&nn->nn_timeout, 0);
         o2net_set_nn_state(nn, sc, 0, 0);
         spin_unlock(&nn->nn_lock);
 
@@ -1922,6 +1999,9 @@ int o2net_init(void)
 
         o2quo_init();
 
+        if (o2net_debugfs_init())
+                return -ENOMEM;
+
         o2net_hand = kzalloc(sizeof(struct o2net_handshake), GFP_KERNEL);
         o2net_keep_req = kzalloc(sizeof(struct o2net_msg), GFP_KERNEL);
         o2net_keep_resp = kzalloc(sizeof(struct o2net_msg), GFP_KERNEL);
@@ -1941,6 +2021,7 @@ int o2net_init(void)
         for (i = 0; i < ARRAY_SIZE(o2net_nodes); i++) {
                 struct o2net_node *nn = o2net_nn_from_num(i);
 
+                atomic_set(&nn->nn_timeout, 0);
                 spin_lock_init(&nn->nn_lock);
                 INIT_DELAYED_WORK(&nn->nn_connect_work, o2net_start_connect);
                 INIT_DELAYED_WORK(&nn->nn_connect_expired,
@@ -1962,4 +2043,5 @@ void o2net_exit(void)
         kfree(o2net_hand);
         kfree(o2net_keep_req);
         kfree(o2net_keep_resp);
+        o2net_debugfs_exit();
 }
diff --git a/fs/ocfs2/cluster/tcp.h b/fs/ocfs2/cluster/tcp.h
index f36f66aab3dd..a705d5d19036 100644
--- a/fs/ocfs2/cluster/tcp.h
+++ b/fs/ocfs2/cluster/tcp.h
@@ -117,4 +117,36 @@ int o2net_num_connected_peers(void);
 int o2net_init(void);
 void o2net_exit(void);
 
+struct o2net_send_tracking;
+struct o2net_sock_container;
+
+#ifdef CONFIG_DEBUG_FS
+int o2net_debugfs_init(void);
+void o2net_debugfs_exit(void);
+void o2net_debug_add_nst(struct o2net_send_tracking *nst);
+void o2net_debug_del_nst(struct o2net_send_tracking *nst);
+void o2net_debug_add_sc(struct o2net_sock_container *sc);
+void o2net_debug_del_sc(struct o2net_sock_container *sc);
+#else
+static int o2net_debugfs_init(void)
+{
+        return 0;
+}
+static void o2net_debugfs_exit(void)
+{
+}
+static void o2net_debug_add_nst(struct o2net_send_tracking *nst)
+{
+}
+static void o2net_debug_del_nst(struct o2net_send_tracking *nst)
+{
+}
+static void o2net_debug_add_sc(struct o2net_sock_container *sc)
+{
+}
+static void o2net_debug_del_sc(struct o2net_sock_container *sc)
+{
+}
+#endif  /* CONFIG_DEBUG_FS */
+
 #endif /* O2CLUSTER_TCP_H */
diff --git a/fs/ocfs2/cluster/tcp_internal.h b/fs/ocfs2/cluster/tcp_internal.h
index d25b9af28500..8d58cfe410b1 100644
--- a/fs/ocfs2/cluster/tcp_internal.h
+++ b/fs/ocfs2/cluster/tcp_internal.h
@@ -95,6 +95,8 @@ struct o2net_node {
         unsigned                        nn_sc_valid:1;
         /* if this is set tx just returns it */
         int                             nn_persistent_error;
+        /* It is only set to 1 after the idle time out. */
+        atomic_t                        nn_timeout;
 
         /* threads waiting for an sc to arrive wait on the wq for generation
          * to increase.  it is increased when a connecting socket succeeds
@@ -164,7 +166,9 @@ struct o2net_sock_container {
         /* original handlers for the sockets */
         void                    (*sc_state_change)(struct sock *sk);
         void                    (*sc_data_ready)(struct sock *sk, int bytes);
-
+#ifdef CONFIG_DEBUG_FS
+        struct list_head        sc_net_debug_item;
+#endif
         struct timeval          sc_tv_timer;
         struct timeval          sc_tv_data_ready;
         struct timeval          sc_tv_advance_start;
@@ -206,4 +210,24 @@ struct o2net_status_wait {
         struct list_head        ns_node_item;
 };
 
+#ifdef CONFIG_DEBUG_FS
+/* just for state dumps */
+struct o2net_send_tracking {
+        struct list_head                st_net_debug_item;
+        struct task_struct              *st_task;
+        struct o2net_sock_container     *st_sc;
+        u32                             st_id;
+        u32                             st_msg_type;
+        u32                             st_msg_key;
+        u8                              st_node;
+        struct timeval                  st_sock_time;
+        struct timeval                  st_send_time;
+        struct timeval                  st_status_time;
+};
+#else
+struct o2net_send_tracking {
+        u32     dummy;
+};
+#endif  /* CONFIG_DEBUG_FS */
+
 #endif /* O2CLUSTER_TCP_INTERNAL_H */