14 files changed, 230 insertions, 147 deletions

diff --git a/fs/notify/dnotify/dnotify.c b/fs/notify/dnotify/dnotify.c
index 63a1ca4b9dee..e2bea2ac5dfb 100644
--- a/fs/notify/dnotify/dnotify.c
+++ b/fs/notify/dnotify/dnotify.c
@@ -79,12 +79,11 @@ static void dnotify_recalc_inode_mask(struct fsnotify_mark *fsn_mark)
  */
 static int dnotify_handle_event(struct fsnotify_group *group,
                                 struct inode *inode,
-                                struct fsnotify_mark *inode_mark,
-                                struct fsnotify_mark *vfsmount_mark,
                                 u32 mask, const void *data, int data_type,
                                 const unsigned char *file_name, u32 cookie,
                                 struct fsnotify_iter_info *iter_info)
 {
+        struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
         struct dnotify_mark *dn_mark;
         struct dnotify_struct *dn;
         struct dnotify_struct **prev;
@@ -95,7 +94,8 @@ static int dnotify_handle_event(struct fsnotify_group *group,
         if (!S_ISDIR(inode->i_mode))
                 return 0;
 
-        BUG_ON(vfsmount_mark);
+        if (WARN_ON(fsnotify_iter_vfsmount_mark(iter_info)))
+                return 0;
 
         dn_mark = container_of(inode_mark, struct dnotify_mark, fsn_mark);
 
@@ -319,7 +319,7 @@ int fcntl_dirnotify(int fd, struct file *filp, unsigned long arg)
                 dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark);
                 spin_lock(&fsn_mark->lock);
         } else {
-                error = fsnotify_add_mark_locked(new_fsn_mark, inode, NULL, 0);
+                error = fsnotify_add_inode_mark_locked(new_fsn_mark, inode, 0);
                 if (error) {
                         mutex_unlock(&dnotify_group->mark_mutex);
                         goto out_err;
diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c
index d94e8031fe5f..f90842efea13 100644
--- a/fs/notify/fanotify/fanotify.c
+++ b/fs/notify/fanotify/fanotify.c
@@ -87,17 +87,17 @@ static int fanotify_get_response(struct fsnotify_group *group,
         return ret;
 }
 
-static bool fanotify_should_send_event(struct fsnotify_mark *inode_mark,
-                                       struct fsnotify_mark *vfsmnt_mark,
-                                       u32 event_mask,
-                                       const void *data, int data_type)
+static bool fanotify_should_send_event(struct fsnotify_iter_info *iter_info,
+                                       u32 event_mask, const void *data,
+                                       int data_type)
 {
         __u32 marks_mask = 0, marks_ignored_mask = 0;
         const struct path *path = data;
+        struct fsnotify_mark *mark;
+        int type;
 
-        pr_debug("%s: inode_mark=%p vfsmnt_mark=%p mask=%x data=%p"
-                 " data_type=%d\n", __func__, inode_mark, vfsmnt_mark,
-                 event_mask, data, data_type);
+        pr_debug("%s: report_mask=%x mask=%x data=%p data_type=%d\n",
+                 __func__, iter_info->report_mask, event_mask, data, data_type);
 
         /* if we don't have enough info to send an event to userspace say no */
         if (data_type != FSNOTIFY_EVENT_PATH)
@@ -108,20 +108,21 @@ static bool fanotify_should_send_event(struct fsnotify_mark *inode_mark,
             !d_can_lookup(path->dentry))
                 return false;
 
-        /*
-         * if the event is for a child and this inode doesn't care about
-         * events on the child, don't send it!
-         */
-        if (inode_mark &&
-            (!(event_mask & FS_EVENT_ON_CHILD) ||
-             (inode_mark->mask & FS_EVENT_ON_CHILD))) {
-                marks_mask |= inode_mark->mask;
-                marks_ignored_mask |= inode_mark->ignored_mask;
-        }
+        fsnotify_foreach_obj_type(type) {
+                if (!fsnotify_iter_should_report_type(iter_info, type))
+                        continue;
+                mark = iter_info->marks[type];
+                /*
+                 * if the event is for a child and this inode doesn't care about
+                 * events on the child, don't send it!
+                 */
+                if (type == FSNOTIFY_OBJ_TYPE_INODE &&
+                    (event_mask & FS_EVENT_ON_CHILD) &&
+                    !(mark->mask & FS_EVENT_ON_CHILD))
+                        continue;
 
-        if (vfsmnt_mark) {
-                marks_mask |= vfsmnt_mark->mask;
-                marks_ignored_mask |= vfsmnt_mark->ignored_mask;
+                marks_mask |= mark->mask;
+                marks_ignored_mask |= mark->ignored_mask;
         }
 
         if (d_is_dir(path->dentry) &&
@@ -178,8 +179,6 @@ init: __maybe_unused
 
 static int fanotify_handle_event(struct fsnotify_group *group,
                                  struct inode *inode,
-                                 struct fsnotify_mark *inode_mark,
-                                 struct fsnotify_mark *fanotify_mark,
                                  u32 mask, const void *data, int data_type,
                                  const unsigned char *file_name, u32 cookie,
                                  struct fsnotify_iter_info *iter_info)
@@ -199,8 +198,7 @@ static int fanotify_handle_event(struct fsnotify_group *group,
         BUILD_BUG_ON(FAN_ACCESS_PERM != FS_ACCESS_PERM);
         BUILD_BUG_ON(FAN_ONDIR != FS_ISDIR);
 
-        if (!fanotify_should_send_event(inode_mark, fanotify_mark, mask, data,
-                                        data_type))
+        if (!fanotify_should_send_event(iter_info, mask, data, data_type))
                 return 0;
 
         pr_debug("%s: group=%p inode=%p mask=%x\n", __func__, group, inode,
diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c
index d478629c728b..10aac1942c9f 100644
--- a/fs/notify/fdinfo.c
+++ b/fs/notify/fdinfo.c
@@ -77,7 +77,7 @@ static void inotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark)
         struct inotify_inode_mark *inode_mark;
         struct inode *inode;
 
-        if (!(mark->connector->flags & FSNOTIFY_OBJ_TYPE_INODE))
+        if (mark->connector->type != FSNOTIFY_OBJ_TYPE_INODE)
                 return;
 
         inode_mark = container_of(mark, struct inotify_inode_mark, fsn_mark);
@@ -116,7 +116,7 @@ static void fanotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark)
         if (mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY)
                 mflags |= FAN_MARK_IGNORED_SURV_MODIFY;
 
-        if (mark->connector->flags & FSNOTIFY_OBJ_TYPE_INODE) {
+        if (mark->connector->type == FSNOTIFY_OBJ_TYPE_INODE) {
                 inode = igrab(mark->connector->inode);
                 if (!inode)
                         return;
@@ -126,7 +126,7 @@ static void fanotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark)
                 show_mark_fhandle(m, inode);
                 seq_putc(m, '\n');
                 iput(inode);
-        } else if (mark->connector->flags & FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
+        } else if (mark->connector->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
                 struct mount *mnt = real_mount(mark->connector->mnt);
 
                 seq_printf(m, "fanotify mnt_id:%x mflags:%x mask:%x ignored_mask:%x\n",
diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c
index 613ec7e5a465..f174397b63a0 100644
--- a/fs/notify/fsnotify.c
+++ b/fs/notify/fsnotify.c
@@ -184,8 +184,6 @@ int __fsnotify_parent(const struct path *path, struct dentry *dentry, __u32 mask
 EXPORT_SYMBOL_GPL(__fsnotify_parent);
 
 static int send_to_group(struct inode *to_tell,
-                         struct fsnotify_mark *inode_mark,
-                         struct fsnotify_mark *vfsmount_mark,
                          __u32 mask, const void *data,
                          int data_is, u32 cookie,
                          const unsigned char *file_name,
@@ -195,48 +193,45 @@ static int send_to_group(struct inode *to_tell,
         __u32 test_mask = (mask & ~FS_EVENT_ON_CHILD);
         __u32 marks_mask = 0;
         __u32 marks_ignored_mask = 0;
+        struct fsnotify_mark *mark;
+        int type;
 
-        if (unlikely(!inode_mark && !vfsmount_mark)) {
-                BUG();
+        if (WARN_ON(!iter_info->report_mask))
                 return 0;
-        }
 
         /* clear ignored on inode modification */
         if (mask & FS_MODIFY) {
-                if (inode_mark &&
-                    !(inode_mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY))
-                        inode_mark->ignored_mask = 0;
-                if (vfsmount_mark &&
-                    !(vfsmount_mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY))
-                        vfsmount_mark->ignored_mask = 0;
-        }
-
-        /* does the inode mark tell us to do something? */
-        if (inode_mark) {
-                group = inode_mark->group;
-                marks_mask |= inode_mark->mask;
-                marks_ignored_mask |= inode_mark->ignored_mask;
+                fsnotify_foreach_obj_type(type) {
+                        if (!fsnotify_iter_should_report_type(iter_info, type))
+                                continue;
+                        mark = iter_info->marks[type];
+                        if (mark &&
+                            !(mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY))
+                                mark->ignored_mask = 0;
+                }
         }
 
-        /* does the vfsmount_mark tell us to do something? */
-        if (vfsmount_mark) {
-                group = vfsmount_mark->group;
-                marks_mask |= vfsmount_mark->mask;
-                marks_ignored_mask |= vfsmount_mark->ignored_mask;
+        fsnotify_foreach_obj_type(type) {
+                if (!fsnotify_iter_should_report_type(iter_info, type))
+                        continue;
+                mark = iter_info->marks[type];
+                /* does the object mark tell us to do something? */
+                if (mark) {
+                        group = mark->group;
+                        marks_mask |= mark->mask;
+                        marks_ignored_mask |= mark->ignored_mask;
+                }
         }
 
-        pr_debug("%s: group=%p to_tell=%p mask=%x inode_mark=%p"
-                 " vfsmount_mark=%p marks_mask=%x marks_ignored_mask=%x"
+        pr_debug("%s: group=%p to_tell=%p mask=%x marks_mask=%x marks_ignored_mask=%x"
                  " data=%p data_is=%d cookie=%d\n",
-                 __func__, group, to_tell, mask, inode_mark, vfsmount_mark,
-                 marks_mask, marks_ignored_mask, data,
-                 data_is, cookie);
+                 __func__, group, to_tell, mask, marks_mask, marks_ignored_mask,
+                 data, data_is, cookie);
 
         if (!(test_mask & marks_mask & ~marks_ignored_mask))
                 return 0;
 
-        return group->ops->handle_event(group, to_tell, inode_mark,
-                                        vfsmount_mark, mask, data, data_is,
+        return group->ops->handle_event(group, to_tell, mask, data, data_is,
                                         file_name, cookie, iter_info);
 }
 
@@ -264,6 +259,57 @@ static struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark)
 }
 
 /*
+ * iter_info is a multi head priority queue of marks.
+ * Pick a subset of marks from queue heads, all with the
+ * same group and set the report_mask for selected subset.
+ * Returns the report_mask of the selected subset.
+ */
+static unsigned int fsnotify_iter_select_report_types(
+                struct fsnotify_iter_info *iter_info)
+{
+        struct fsnotify_group *max_prio_group = NULL;
+        struct fsnotify_mark *mark;
+        int type;
+
+        /* Choose max prio group among groups of all queue heads */
+        fsnotify_foreach_obj_type(type) {
+                mark = iter_info->marks[type];
+                if (mark &&
+                    fsnotify_compare_groups(max_prio_group, mark->group) > 0)
+                        max_prio_group = mark->group;
+        }
+
+        if (!max_prio_group)
+                return 0;
+
+        /* Set the report mask for marks from same group as max prio group */
+        iter_info->report_mask = 0;
+        fsnotify_foreach_obj_type(type) {
+                mark = iter_info->marks[type];
+                if (mark &&
+                    fsnotify_compare_groups(max_prio_group, mark->group) == 0)
+                        fsnotify_iter_set_report_type(iter_info, type);
+        }
+
+        return iter_info->report_mask;
+}
+
+/*
+ * Pop from iter_info multi head queue, the marks that were iterated in the
+ * current iteration step.
+ */
+static void fsnotify_iter_next(struct fsnotify_iter_info *iter_info)
+{
+        int type;
+
+        fsnotify_foreach_obj_type(type) {
+                if (fsnotify_iter_should_report_type(iter_info, type))
+                        iter_info->marks[type] =
+                                fsnotify_next_mark(iter_info->marks[type]);
+        }
+}
+
+/*
  * This is the main call to fsnotify.  The VFS calls into hook specific functions
  * in linux/fsnotify.h.  Those functions then in turn call here.  Here will call
  * out to all of the registered fsnotify_group.  Those groups can then use the
@@ -307,15 +353,15 @@ int fsnotify(struct inode *to_tell, __u32 mask, const void *data, int data_is,
 
         if ((mask & FS_MODIFY) ||
             (test_mask & to_tell->i_fsnotify_mask)) {
-                iter_info.inode_mark =
+                iter_info.marks[FSNOTIFY_OBJ_TYPE_INODE] =
                         fsnotify_first_mark(&to_tell->i_fsnotify_marks);
         }
 
         if (mnt && ((mask & FS_MODIFY) ||
                     (test_mask & mnt->mnt_fsnotify_mask))) {
-                iter_info.inode_mark =
+                iter_info.marks[FSNOTIFY_OBJ_TYPE_INODE] =
                         fsnotify_first_mark(&to_tell->i_fsnotify_marks);
-                iter_info.vfsmount_mark =
+                iter_info.marks[FSNOTIFY_OBJ_TYPE_VFSMOUNT] =
                         fsnotify_first_mark(&mnt->mnt_fsnotify_marks);
         }
 
@@ -324,32 +370,14 @@ int fsnotify(struct inode *to_tell, __u32 mask, const void *data, int data_is,
          * ignore masks are properly reflected for mount mark notifications.
          * That's why this traversal is so complicated...
          */
-        while (iter_info.inode_mark || iter_info.vfsmount_mark) {
-                struct fsnotify_mark *inode_mark = iter_info.inode_mark;
-                struct fsnotify_mark *vfsmount_mark = iter_info.vfsmount_mark;
-
-                if (inode_mark && vfsmount_mark) {
-                        int cmp = fsnotify_compare_groups(inode_mark->group,
-                                                          vfsmount_mark->group);
-                        if (cmp > 0)
-                                inode_mark = NULL;
-                        else if (cmp < 0)
-                                vfsmount_mark = NULL;
-                }
-
-                ret = send_to_group(to_tell, inode_mark, vfsmount_mark, mask,
-                                    data, data_is, cookie, file_name,
-                                    &iter_info);
+        while (fsnotify_iter_select_report_types(&iter_info)) {
+                ret = send_to_group(to_tell, mask, data, data_is, cookie,
+                                    file_name, &iter_info);
 
                 if (ret && (mask & ALL_FSNOTIFY_PERM_EVENTS))
                         goto out;
 
-                if (inode_mark)
-                        iter_info.inode_mark =
-                                fsnotify_next_mark(iter_info.inode_mark);
-                if (vfsmount_mark)
-                        iter_info.vfsmount_mark =
-                                fsnotify_next_mark(iter_info.vfsmount_mark);
+                fsnotify_iter_next(&iter_info);
         }
         ret = 0;
 out:
diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h
index 60f365dc1408..34515d2c4ba3 100644
--- a/fs/notify/fsnotify.h
+++ b/fs/notify/fsnotify.h
@@ -9,12 +9,6 @@
 
 #include "../mount.h"
 
-struct fsnotify_iter_info {
-        struct fsnotify_mark *inode_mark;
-        struct fsnotify_mark *vfsmount_mark;
-        int srcu_idx;
-};
-
 /* destroy all events sitting in this groups notification queue */
 extern void fsnotify_flush_notify(struct fsnotify_group *group);
 
diff --git a/fs/notify/group.c b/fs/notify/group.c
index b7a4b6a69efa..aa5468f23e45 100644
--- a/fs/notify/group.c
+++ b/fs/notify/group.c
@@ -67,7 +67,7 @@ void fsnotify_destroy_group(struct fsnotify_group *group)
         fsnotify_group_stop_queueing(group);
 
         /* Clear all marks for this group and queue them for destruction */
-        fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_ALL_TYPES);
+        fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_ALL_TYPES_MASK);
 
         /*
          * Some marks can still be pinned when waiting for response from
diff --git a/fs/notify/inotify/inotify.h b/fs/notify/inotify/inotify.h
index c00d2caca894..7e4578d35b61 100644
--- a/fs/notify/inotify/inotify.h
+++ b/fs/notify/inotify/inotify.h
@@ -25,8 +25,6 @@ extern void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark,
                                            struct fsnotify_group *group);
 extern int inotify_handle_event(struct fsnotify_group *group,
                                 struct inode *inode,
-                                struct fsnotify_mark *inode_mark,
-                                struct fsnotify_mark *vfsmount_mark,
                                 u32 mask, const void *data, int data_type,
                                 const unsigned char *file_name, u32 cookie,
                                 struct fsnotify_iter_info *iter_info);
diff --git a/fs/notify/inotify/inotify_fsnotify.c b/fs/notify/inotify/inotify_fsnotify.c
index 40dedb37a1f3..9ab6dde38a14 100644
--- a/fs/notify/inotify/inotify_fsnotify.c
+++ b/fs/notify/inotify/inotify_fsnotify.c
@@ -65,12 +65,11 @@ static int inotify_merge(struct list_head *list,
 
 int inotify_handle_event(struct fsnotify_group *group,
                          struct inode *inode,
-                         struct fsnotify_mark *inode_mark,
-                         struct fsnotify_mark *vfsmount_mark,
                          u32 mask, const void *data, int data_type,
                          const unsigned char *file_name, u32 cookie,
                          struct fsnotify_iter_info *iter_info)
 {
+        struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
         struct inotify_inode_mark *i_mark;
         struct inotify_event_info *event;
         struct fsnotify_event *fsn_event;
@@ -78,7 +77,8 @@ int inotify_handle_event(struct fsnotify_group *group,
         int len = 0;
         int alloc_len = sizeof(struct inotify_event_info);
 
-        BUG_ON(vfsmount_mark);
+        if (WARN_ON(fsnotify_iter_vfsmount_mark(iter_info)))
+                return 0;
 
         if ((inode_mark->mask & FS_EXCL_UNLINK) &&
             (data_type == FSNOTIFY_EVENT_PATH)) {
diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c
index ef32f3657958..1cf5b779d862 100644
--- a/fs/notify/inotify/inotify_user.c
+++ b/fs/notify/inotify/inotify_user.c
@@ -485,10 +485,14 @@ void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark,
                                     struct fsnotify_group *group)
 {
         struct inotify_inode_mark *i_mark;
+        struct fsnotify_iter_info iter_info = { };
+
+        fsnotify_iter_set_report_type_mark(&iter_info, FSNOTIFY_OBJ_TYPE_INODE,
+                                           fsn_mark);
 
         /* Queue ignore event for the watch */
-        inotify_handle_event(group, NULL, fsn_mark, NULL, FS_IN_IGNORED,
-                             NULL, FSNOTIFY_EVENT_NONE, NULL, 0, NULL);
+        inotify_handle_event(group, NULL, FS_IN_IGNORED, NULL,
+                             FSNOTIFY_EVENT_NONE, NULL, 0, &iter_info);
 
         i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark);
         /* remove this mark from the idr */
@@ -578,7 +582,7 @@ static int inotify_new_watch(struct fsnotify_group *group,
         }
 
         /* we are on the idr, now get on the inode */
-        ret = fsnotify_add_mark_locked(&tmp_i_mark->fsn_mark, inode, NULL, 0);
+        ret = fsnotify_add_inode_mark_locked(&tmp_i_mark->fsn_mark, inode, 0);
         if (ret) {
                 /* we failed to get on the inode, get off the idr */
                 inotify_remove_from_idr(group, tmp_i_mark);
diff --git a/fs/notify/mark.c b/fs/notify/mark.c
index e9191b416434..61f4c5fa34c7 100644
--- a/fs/notify/mark.c
+++ b/fs/notify/mark.c
@@ -119,9 +119,9 @@ static void __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
                 if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)
                         new_mask |= mark->mask;
         }
-        if (conn->flags & FSNOTIFY_OBJ_TYPE_INODE)
+        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
                 conn->inode->i_fsnotify_mask = new_mask;
-        else if (conn->flags & FSNOTIFY_OBJ_TYPE_VFSMOUNT)
+        else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)
                 real_mount(conn->mnt)->mnt_fsnotify_mask = new_mask;
 }
 
@@ -139,7 +139,7 @@ void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
         spin_lock(&conn->lock);
         __fsnotify_recalc_mask(conn);
         spin_unlock(&conn->lock);
-        if (conn->flags & FSNOTIFY_OBJ_TYPE_INODE)
+        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
                 __fsnotify_update_child_dentry_flags(conn->inode);
 }
 
@@ -166,18 +166,18 @@ static struct inode *fsnotify_detach_connector_from_object(
 {
         struct inode *inode = NULL;
 
-        if (conn->flags & FSNOTIFY_OBJ_TYPE_INODE) {
+        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
                 inode = conn->inode;
                 rcu_assign_pointer(inode->i_fsnotify_marks, NULL);
                 inode->i_fsnotify_mask = 0;
                 conn->inode = NULL;
-                conn->flags &= ~FSNOTIFY_OBJ_TYPE_INODE;
-        } else if (conn->flags & FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
+                conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
+        } else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
                 rcu_assign_pointer(real_mount(conn->mnt)->mnt_fsnotify_marks,
                                    NULL);
                 real_mount(conn->mnt)->mnt_fsnotify_mask = 0;
                 conn->mnt = NULL;
-                conn->flags &= ~FSNOTIFY_OBJ_TYPE_VFSMOUNT;
+                conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
         }
 
         return inode;
@@ -294,12 +294,12 @@ static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)
 
 bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
 {
-        /* This can fail if mark is being removed */
-        if (!fsnotify_get_mark_safe(iter_info->inode_mark))
-                return false;
-        if (!fsnotify_get_mark_safe(iter_info->vfsmount_mark)) {
-                fsnotify_put_mark_wake(iter_info->inode_mark);
-                return false;
+        int type;
+
+        fsnotify_foreach_obj_type(type) {
+                /* This can fail if mark is being removed */
+                if (!fsnotify_get_mark_safe(iter_info->marks[type]))
+                        goto fail;
         }
 
         /*
@@ -310,13 +310,20 @@ bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
         srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx);
 
         return true;
+
+fail:
+        for (type--; type >= 0; type--)
+                fsnotify_put_mark_wake(iter_info->marks[type]);
+        return false;
 }
 
 void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info)
 {
+        int type;
+
         iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
-        fsnotify_put_mark_wake(iter_info->inode_mark);
-        fsnotify_put_mark_wake(iter_info->vfsmount_mark);
+        fsnotify_foreach_obj_type(type)
+                fsnotify_put_mark_wake(iter_info->marks[type]);
 }
 
 /*
@@ -442,10 +449,10 @@ static int fsnotify_attach_connector_to_object(
         spin_lock_init(&conn->lock);
         INIT_HLIST_HEAD(&conn->list);
         if (inode) {
-                conn->flags = FSNOTIFY_OBJ_TYPE_INODE;
+                conn->type = FSNOTIFY_OBJ_TYPE_INODE;
                 conn->inode = igrab(inode);
         } else {
-                conn->flags = FSNOTIFY_OBJ_TYPE_VFSMOUNT;
+                conn->type = FSNOTIFY_OBJ_TYPE_VFSMOUNT;
                 conn->mnt = mnt;
         }
         /*
@@ -479,8 +486,7 @@ static struct fsnotify_mark_connector *fsnotify_grab_connector(
         if (!conn)
                 goto out;
         spin_lock(&conn->lock);
-        if (!(conn->flags & (FSNOTIFY_OBJ_TYPE_INODE |
-                             FSNOTIFY_OBJ_TYPE_VFSMOUNT))) {
+        if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {
                 spin_unlock(&conn->lock);
                 srcu_read_unlock(&fsnotify_mark_srcu, idx);
                 return NULL;
@@ -646,16 +652,16 @@ struct fsnotify_mark *fsnotify_find_mark(
         return NULL;
 }
 
-/* Clear any marks in a group with given type */
+/* Clear any marks in a group with given type mask */
 void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
-                                   unsigned int type)
+                                   unsigned int type_mask)
 {
         struct fsnotify_mark *lmark, *mark;
         LIST_HEAD(to_free);
         struct list_head *head = &to_free;
 
         /* Skip selection step if we want to clear all marks. */
-        if (type == FSNOTIFY_OBJ_ALL_TYPES) {
+        if (type_mask == FSNOTIFY_OBJ_ALL_TYPES_MASK) {
                 head = &group->marks_list;
                 goto clear;
         }
@@ -670,7 +676,7 @@ void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
          */
         mutex_lock_nested(&group->mark_mutex, SINGLE_DEPTH_NESTING);
         list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {
-                if (mark->connector->flags & type)
+                if ((1U << mark->connector->type) & type_mask)
                         list_move(&mark->g_list, &to_free);
         }
         mutex_unlock(&group->mark_mutex);
diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
index e64c0294f50b..b38964a7a521 100644
--- a/include/linux/fsnotify_backend.h
+++ b/include/linux/fsnotify_backend.h
@@ -98,8 +98,6 @@ struct fsnotify_iter_info;
 struct fsnotify_ops {
         int (*handle_event)(struct fsnotify_group *group,
                             struct inode *inode,
-                            struct fsnotify_mark *inode_mark,
-                            struct fsnotify_mark *vfsmount_mark,
                             u32 mask, const void *data, int data_type,
                             const unsigned char *file_name, u32 cookie,
                             struct fsnotify_iter_info *iter_info);
@@ -201,6 +199,57 @@ struct fsnotify_group {
 #define FSNOTIFY_EVENT_PATH     1
 #define FSNOTIFY_EVENT_INODE    2
 
+enum fsnotify_obj_type {
+        FSNOTIFY_OBJ_TYPE_INODE,
+        FSNOTIFY_OBJ_TYPE_VFSMOUNT,
+        FSNOTIFY_OBJ_TYPE_COUNT,
+        FSNOTIFY_OBJ_TYPE_DETACHED = FSNOTIFY_OBJ_TYPE_COUNT
+};
+
+#define FSNOTIFY_OBJ_TYPE_INODE_FL      (1U << FSNOTIFY_OBJ_TYPE_INODE)
+#define FSNOTIFY_OBJ_TYPE_VFSMOUNT_FL   (1U << FSNOTIFY_OBJ_TYPE_VFSMOUNT)
+#define FSNOTIFY_OBJ_ALL_TYPES_MASK     ((1U << FSNOTIFY_OBJ_TYPE_COUNT) - 1)
+
+struct fsnotify_iter_info {
+        struct fsnotify_mark *marks[FSNOTIFY_OBJ_TYPE_COUNT];
+        unsigned int report_mask;
+        int srcu_idx;
+};
+
+static inline bool fsnotify_iter_should_report_type(
+                struct fsnotify_iter_info *iter_info, int type)
+{
+        return (iter_info->report_mask & (1U << type));
+}
+
+static inline void fsnotify_iter_set_report_type(
+                struct fsnotify_iter_info *iter_info, int type)
+{
+        iter_info->report_mask |= (1U << type);
+}
+
+static inline void fsnotify_iter_set_report_type_mark(
+                struct fsnotify_iter_info *iter_info, int type,
+                struct fsnotify_mark *mark)
+{
+        iter_info->marks[type] = mark;
+        iter_info->report_mask |= (1U << type);
+}
+
+#define FSNOTIFY_ITER_FUNCS(name, NAME) \
+static inline struct fsnotify_mark *fsnotify_iter_##name##_mark( \
+                struct fsnotify_iter_info *iter_info) \
+{ \
+        return (iter_info->report_mask & FSNOTIFY_OBJ_TYPE_##NAME##_FL) ? \
+                iter_info->marks[FSNOTIFY_OBJ_TYPE_##NAME] : NULL; \
+}
+
+FSNOTIFY_ITER_FUNCS(inode, INODE)
+FSNOTIFY_ITER_FUNCS(vfsmount, VFSMOUNT)
+
+#define fsnotify_foreach_obj_type(type) \
+        for (type = 0; type < FSNOTIFY_OBJ_TYPE_COUNT; type++)
+
 /*
  * Inode / vfsmount point to this structure which tracks all marks attached to
  * the inode / vfsmount. The reference to inode / vfsmount is held by this
@@ -209,11 +258,7 @@ struct fsnotify_group {
  */
 struct fsnotify_mark_connector {
         spinlock_t lock;
-#define FSNOTIFY_OBJ_TYPE_INODE         0x01
-#define FSNOTIFY_OBJ_TYPE_VFSMOUNT      0x02
-#define FSNOTIFY_OBJ_ALL_TYPES          (FSNOTIFY_OBJ_TYPE_INODE | \
-                                         FSNOTIFY_OBJ_TYPE_VFSMOUNT)
-        unsigned int flags;     /* Type of object [lock] */
+        unsigned int type;      /* Type of object [lock] */
         union { /* Object pointer [lock] */
                 struct inode *inode;
                 struct vfsmount *mnt;
@@ -356,7 +401,21 @@ extern struct fsnotify_mark *fsnotify_find_mark(
 extern int fsnotify_add_mark(struct fsnotify_mark *mark, struct inode *inode,
                              struct vfsmount *mnt, int allow_dups);
 extern int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
-                                    struct inode *inode, struct vfsmount *mnt, int allow_dups);
+                                    struct inode *inode, struct vfsmount *mnt,
+                                    int allow_dups);
+/* attach the mark to the inode */
+static inline int fsnotify_add_inode_mark(struct fsnotify_mark *mark,
+                                          struct inode *inode,
+                                          int allow_dups)
+{
+        return fsnotify_add_mark(mark, inode, NULL, allow_dups);
+}
+static inline int fsnotify_add_inode_mark_locked(struct fsnotify_mark *mark,
+                                                 struct inode *inode,
+                                                 int allow_dups)
+{
+        return fsnotify_add_mark_locked(mark, inode, NULL, allow_dups);
+}
 /* given a group and a mark, flag mark to be freed when all references are dropped */
 extern void fsnotify_destroy_mark(struct fsnotify_mark *mark,
                                   struct fsnotify_group *group);
@@ -369,12 +428,12 @@ extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group, unsigned
 /* run all the marks in a group, and clear all of the vfsmount marks */
 static inline void fsnotify_clear_vfsmount_marks_by_group(struct fsnotify_group *group)
 {
-        fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_VFSMOUNT);
+        fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_VFSMOUNT_FL);
 }
 /* run all the marks in a group, and clear all of the inode marks */
 static inline void fsnotify_clear_inode_marks_by_group(struct fsnotify_group *group)
 {
-        fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_INODE);
+        fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_INODE_FL);
 }
 extern void fsnotify_get_mark(struct fsnotify_mark *mark);
 extern void fsnotify_put_mark(struct fsnotify_mark *mark);
diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c
index 52f368b6561e..fba78047fb37 100644
--- a/kernel/audit_fsnotify.c
+++ b/kernel/audit_fsnotify.c
@@ -109,7 +109,7 @@ struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pa
         audit_update_mark(audit_mark, dentry->d_inode);
         audit_mark->rule = krule;
 
-        ret = fsnotify_add_mark(&audit_mark->mark, inode, NULL, true);
+        ret = fsnotify_add_inode_mark(&audit_mark->mark, inode, true);
         if (ret < 0) {
                 fsnotify_put_mark(&audit_mark->mark);
                 audit_mark = ERR_PTR(ret);
@@ -165,12 +165,11 @@ static void audit_autoremove_mark_rule(struct audit_fsnotify_mark *audit_mark)
 /* Update mark data in audit rules based on fsnotify events. */
 static int audit_mark_handle_event(struct fsnotify_group *group,
                                     struct inode *to_tell,
-                                    struct fsnotify_mark *inode_mark,
-                                    struct fsnotify_mark *vfsmount_mark,
                                     u32 mask, const void *data, int data_type,
                                     const unsigned char *dname, u32 cookie,
                                     struct fsnotify_iter_info *iter_info)
 {
+        struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
         struct audit_fsnotify_mark *audit_mark;
         const struct inode *inode = NULL;
 
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 67e6956c0b61..c99ebaae5abc 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -288,8 +288,8 @@ static void untag_chunk(struct node *p)
         if (!new)
                 goto Fallback;
 
-        if (fsnotify_add_mark_locked(&new->mark, entry->connector->inode,
-                                     NULL, 1)) {
+        if (fsnotify_add_inode_mark_locked(&new->mark, entry->connector->inode,
+                                           1)) {
                 fsnotify_put_mark(&new->mark);
                 goto Fallback;
         }
@@ -354,7 +354,7 @@ static int create_chunk(struct inode *inode, struct audit_tree *tree)
                 return -ENOMEM;
 
         entry = &chunk->mark;
-        if (fsnotify_add_mark(entry, inode, NULL, 0)) {
+        if (fsnotify_add_inode_mark(entry, inode, 0)) {
                 fsnotify_put_mark(entry);
                 return -ENOSPC;
         }
@@ -434,8 +434,8 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree)
                 return -ENOENT;
         }
 
-        if (fsnotify_add_mark_locked(chunk_entry,
-                             old_entry->connector->inode, NULL, 1)) {
+        if (fsnotify_add_inode_mark_locked(chunk_entry,
+                             old_entry->connector->inode, 1)) {
                 spin_unlock(&old_entry->lock);
                 mutex_unlock(&old_entry->group->mark_mutex);
                 fsnotify_put_mark(chunk_entry);
@@ -989,8 +989,6 @@ static void evict_chunk(struct audit_chunk *chunk)
 
 static int audit_tree_handle_event(struct fsnotify_group *group,
                                    struct inode *to_tell,
-                                   struct fsnotify_mark *inode_mark,
-                                   struct fsnotify_mark *vfsmount_mark,
                                    u32 mask, const void *data, int data_type,
                                    const unsigned char *file_name, u32 cookie,
                                    struct fsnotify_iter_info *iter_info)
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
index f1ba88994508..c17c0c268436 100644
--- a/kernel/audit_watch.c
+++ b/kernel/audit_watch.c
@@ -160,7 +160,7 @@ static struct audit_parent *audit_init_parent(struct path *path)
 
         fsnotify_init_mark(&parent->mark, audit_watch_group);
         parent->mark.mask = AUDIT_FS_WATCH;
-        ret = fsnotify_add_mark(&parent->mark, inode, NULL, 0);
+        ret = fsnotify_add_inode_mark(&parent->mark, inode, 0);
         if (ret < 0) {
                 audit_free_parent(parent);
                 return ERR_PTR(ret);
@@ -472,12 +472,11 @@ void audit_remove_watch_rule(struct audit_krule *krule)
 /* Update watch data in audit rules based on fsnotify events. */
 static int audit_watch_handle_event(struct fsnotify_group *group,
                                     struct inode *to_tell,
-                                    struct fsnotify_mark *inode_mark,
-                                    struct fsnotify_mark *vfsmount_mark,
                                     u32 mask, const void *data, int data_type,
                                     const unsigned char *dname, u32 cookie,
                                     struct fsnotify_iter_info *iter_info)
 {
+        struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info);
         const struct inode *inode;
         struct audit_parent *parent;