ceph: fix cap removal races

The iterate_session_caps helper traverses the session caps list and tries
to grab an inode reference.  However, the __ceph_remove_cap was clearing
the inode backpointer _before_ removing itself from the session list,
causing a null pointer dereference.

Clear cap->ci under protection of s_cap_lock to avoid the race, and to
tightly couple the list and backpointer state.  Use a local flag to
indicate whether we are releasing the cap, as cap->session may be modified
by a racing thread in iterate_session_caps.

Signed-off-by: Sage Weil <sage@newdream.net>

2 files changed, 15 insertions, 9 deletions

diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
index 0c1681806867..d9400534b279 100644
--- a/fs/ceph/caps.c
+++ b/fs/ceph/caps.c
@@ -858,6 +858,8 @@ static int __ceph_is_any_caps(struct ceph_inode_info *ci)
 }
 
 /*
+ * Remove a cap.  Take steps to deal with a racing iterate_session_caps.
+ *
  * caller should hold i_lock.
  * caller will not hold session s_mutex if called from destroy_inode.
  */
@@ -866,15 +868,10 @@ void __ceph_remove_cap(struct ceph_cap *cap)
         struct ceph_mds_session *session = cap->session;
         struct ceph_inode_info *ci = cap->ci;
         struct ceph_mds_client *mdsc = &ceph_client(ci->vfs_inode.i_sb)->mdsc;
+        int removed = 0;
 
         dout("__ceph_remove_cap %p from %p\n", cap, &ci->vfs_inode);
 
-        /* remove from inode list */
-        rb_erase(&cap->ci_node, &ci->i_caps);
-        cap->ci = NULL;
-        if (ci->i_auth_cap == cap)
-                ci->i_auth_cap = NULL;
-
         /* remove from session list */
         spin_lock(&session->s_cap_lock);
         if (session->s_cap_iterator == cap) {
@@ -885,10 +882,18 @@ void __ceph_remove_cap(struct ceph_cap *cap)
                 list_del_init(&cap->session_caps);
                 session->s_nr_caps--;
                 cap->session = NULL;
+                removed = 1;
         }
+        /* protect backpointer with s_cap_lock: see iterate_session_caps */
+        cap->ci = NULL;
         spin_unlock(&session->s_cap_lock);
 
-        if (cap->session == NULL)
+        /* remove from inode list */
+        rb_erase(&cap->ci_node, &ci->i_caps);
+        if (ci->i_auth_cap == cap)
+                ci->i_auth_cap = NULL;
+
+        if (removed)
                 ceph_put_cap(cap);
 
         if (!__ceph_is_any_caps(ci) && ci->i_snap_realm) {
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index eccc0ecad1a2..24561a557e01 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -736,9 +736,10 @@ static void cleanup_cap_releases(struct ceph_mds_session *session)
 }
 
 /*
- * Helper to safely iterate over all caps associated with a session.
+ * Helper to safely iterate over all caps associated with a session, with
+ * special care taken to handle a racing __ceph_remove_cap().
  *
- * caller must hold session s_mutex
+ * Caller must hold session s_mutex.
  */
 static int iterate_session_caps(struct ceph_mds_session *session,
                                  int (*cb)(struct inode *, struct ceph_cap *,