GFS2: update freeze code to use freeze/thaw_super on all nodes

The current gfs2 freezing code is considerably more complicated than it
should be because it doesn't use the vfs freezing code on any node except
the one that begins the freeze.  This is because it needs to acquire a
cluster glock before calling the vfs code to prevent a deadlock, and
without the new freeze_super and thaw_super hooks, that was impossible. To
deal with the issue, gfs2 had to do some hacky locking tricks to make sure
that a frozen node couldn't be holding on a lock it needed to do the
unfreeze ioctl.

This patch makes use of the new hooks to simply the gfs2 locking code. Now,
all the nodes in the cluster freeze and thaw in exactly the same way. Every
node in the cluster caches the freeze glock in the shared state.  The new
freeze_super hook allows the freezing node to grab this freeze glock in
the exclusive state without first calling the vfs freeze_super function.
All the nodes in the cluster see this lock change, and call the vfs
freeze_super function. The vfs locking code guarantees that the nodes can't
get stuck holding the glocks necessary to unfreeze the system.  To
unfreeze, the freezing node uses the new thaw_super hook to drop the freeze
glock. Again, all the nodes notice this, reacquire the glock in shared mode
and call the vfs thaw_super function.

Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
Signed-off-by: Steven Whitehouse <swhiteho@redhat.com>

10 files changed, 161 insertions, 126 deletions

diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c
index 1cc0bba6313f..fe91951c3361 100644
--- a/fs/gfs2/glops.c
+++ b/fs/gfs2/glops.c
@@ -28,6 +28,8 @@
 #include "trans.h"
 #include "dir.h"
 
+struct workqueue_struct *gfs2_freeze_wq;
+
 static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh)
 {
         fs_err(gl->gl_sbd, "AIL buffer %p: blocknr %llu state 0x%08lx mapping %p page state 0x%lx\n",
@@ -94,11 +96,8 @@ static void gfs2_ail_empty_gl(struct gfs2_glock *gl)
          * on the stack */
         tr.tr_reserved = 1 + gfs2_struct2blk(sdp, tr.tr_revokes, sizeof(u64));
         tr.tr_ip = _RET_IP_;
-        sb_start_intwrite(sdp->sd_vfs);
-        if (gfs2_log_reserve(sdp, tr.tr_reserved) < 0) {
-                sb_end_intwrite(sdp->sd_vfs);
+        if (gfs2_log_reserve(sdp, tr.tr_reserved) < 0)
                 return;
-        }
         WARN_ON_ONCE(current->journal_info);
         current->journal_info = &tr;
 
@@ -469,20 +468,19 @@ static void inode_go_dump(struct seq_file *seq, const struct gfs2_glock *gl)
 
 static void freeze_go_sync(struct gfs2_glock *gl)
 {
+        int error = 0;
         struct gfs2_sbd *sdp = gl->gl_sbd;
-        DEFINE_WAIT(wait);
 
         if (gl->gl_state == LM_ST_SHARED &&
             test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) {
-                atomic_set(&sdp->sd_log_freeze, 1);
-                wake_up(&sdp->sd_logd_waitq);
-                do {
-                        prepare_to_wait(&sdp->sd_log_frozen_wait, &wait,
-                                        TASK_UNINTERRUPTIBLE);
-                        if (atomic_read(&sdp->sd_log_freeze))
-                                io_schedule();
-                } while(atomic_read(&sdp->sd_log_freeze));
-                finish_wait(&sdp->sd_log_frozen_wait, &wait);
+                atomic_set(&sdp->sd_freeze_state, SFS_STARTING_FREEZE);
+                error = freeze_super(sdp->sd_vfs);
+                if (error) {
+                        printk(KERN_INFO "GFS2: couldn't freeze filesystem: %d\n", error);
+                        gfs2_assert_withdraw(sdp, 0);
+                }
+                queue_work(gfs2_freeze_wq, &sdp->sd_freeze_work);
+                gfs2_log_flush(sdp, NULL, FREEZE_FLUSH);
         }
 }
 
diff --git a/fs/gfs2/glops.h b/fs/gfs2/glops.h
index 7455d2629bcb..8ed1857c1a8d 100644
--- a/fs/gfs2/glops.h
+++ b/fs/gfs2/glops.h
@@ -12,6 +12,8 @@
 
 #include "incore.h"
 
+extern struct workqueue_struct *gfs2_freeze_wq;
+
 extern const struct gfs2_glock_operations gfs2_meta_glops;
 extern const struct gfs2_glock_operations gfs2_inode_glops;
 extern const struct gfs2_glock_operations gfs2_rgrp_glops;
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index 1b899187be5a..7a2dbbc0d634 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -588,6 +588,12 @@ enum {
         SDF_SKIP_DLM_UNLOCK     = 8,
 };
 
+enum gfs2_freeze_state {
+        SFS_UNFROZEN            = 0,
+        SFS_STARTING_FREEZE     = 1,
+        SFS_FROZEN              = 2,
+};
+
 #define GFS2_FSNAME_LEN         256
 
 struct gfs2_inum_host {
@@ -685,6 +691,7 @@ struct gfs2_sbd {
         struct gfs2_holder sd_live_gh;
         struct gfs2_glock *sd_rename_gl;
         struct gfs2_glock *sd_freeze_gl;
+        struct work_struct sd_freeze_work;
         wait_queue_head_t sd_glock_wait;
         atomic_t sd_glock_disposal;
         struct completion sd_locking_init;
@@ -789,6 +796,9 @@ struct gfs2_sbd {
         wait_queue_head_t sd_log_flush_wait;
         int sd_log_error;
 
+        atomic_t sd_reserving_log;
+        wait_queue_head_t sd_reserving_log_wait;
+
         unsigned int sd_log_flush_head;
         u64 sd_log_flush_wrapped;
 
@@ -798,12 +808,8 @@ struct gfs2_sbd {
 
         /* For quiescing the filesystem */
         struct gfs2_holder sd_freeze_gh;
-        struct gfs2_holder sd_freeze_root_gh;
-        struct gfs2_holder sd_thaw_gh;
-        atomic_t sd_log_freeze;
-        atomic_t sd_frozen_root;
-        wait_queue_head_t sd_frozen_root_wait;
-        wait_queue_head_t sd_log_frozen_wait;
+        atomic_t sd_freeze_state;
+        struct mutex sd_freeze_mutex;
 
         char sd_fsname[GFS2_FSNAME_LEN];
         char sd_table_name[GFS2_FSNAME_LEN];
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index b41b5c7898da..04065e5af4b6 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -1618,26 +1618,18 @@ int gfs2_permission(struct inode *inode, int mask)
 {
         struct gfs2_inode *ip;
         struct gfs2_holder i_gh;
-        struct gfs2_sbd *sdp = GFS2_SB(inode);
         int error;
         int unlock = 0;
-        int frozen_root = 0;
 
 
         ip = GFS2_I(inode);
         if (gfs2_glock_is_locked_by_me(ip->i_gl) == NULL) {
-                if (unlikely(gfs2_glock_is_held_excl(sdp->sd_freeze_gl) &&
-                             inode == sdp->sd_root_dir->d_inode &&
-                             atomic_inc_not_zero(&sdp->sd_frozen_root)))
-                        frozen_root = 1;
-                else {
-                        if (mask & MAY_NOT_BLOCK)
-                                return -ECHILD;
-                        error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
-                        if (error)
-                                return error;
-                        unlock = 1;
-                }
+                if (mask & MAY_NOT_BLOCK)
+                        return -ECHILD;
+                error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
+                if (error)
+                        return error;
+                unlock = 1;
         }
 
         if ((mask & MAY_WRITE) && IS_IMMUTABLE(inode))
@@ -1646,8 +1638,6 @@ int gfs2_permission(struct inode *inode, int mask)
                 error = generic_permission(inode, mask);
         if (unlock)
                 gfs2_glock_dq_uninit(&i_gh);
-        else if (frozen_root && atomic_dec_and_test(&sdp->sd_frozen_root))
-                wake_up(&sdp->sd_frozen_root_wait);
 
         return error;
 }
@@ -1820,29 +1810,19 @@ static int gfs2_getattr(struct vfsmount *mnt, struct dentry *dentry,
         struct inode *inode = dentry->d_inode;