aio: give shared kioctx fields their own cachelines

[akpm@linux-foundation.org: make reqs_active __cacheline_aligned_in_smp]
Signed-off-by: Kent Overstreet <koverstreet@google.com>
Cc: Zach Brown <zab@redhat.com>
Cc: Felipe Balbi <balbi@ti.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Mark Fasheh <mfasheh@suse.com>
Cc: Joel Becker <jlbec@evilplan.org>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Asai Thambi S P <asamymuthupa@micron.com>
Cc: Selvan Mani <smani@micron.com>
Cc: Sam Bradshaw <sbradshaw@micron.com>
Cc: Jeff Moyer <jmoyer@redhat.com>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Benjamin LaHaise <bcrl@kvack.org>
Reviewed-by: "Theodore Ts'o" <tytso@mit.edu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 15 insertions, 12 deletions

diff --git a/fs/aio.c b/fs/aio.c
index 16239fbd6458..986ff305a856 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -67,13 +67,6 @@ struct kioctx {
         unsigned long           user_id;
         struct hlist_node       list;
 
-        wait_queue_head_t       wait;
-
-        spinlock_t              ctx_lock;
-
-        atomic_t                reqs_active;
-        struct list_head        active_reqs;    /* used for cancellation */
-
         /*
          * This is what userspace passed to io_setup(), it's not used for
          * anything but counting against the global max_reqs quota.
@@ -92,19 +85,29 @@ struct kioctx {
         struct page             **ring_pages;
         long                    nr_pages;
 
+        struct rcu_head         rcu_head;
+        struct work_struct      rcu_work;
+
+        struct {
+                atomic_t        reqs_active;
+        } ____cacheline_aligned_in_smp;
+
+        struct {
+                spinlock_t      ctx_lock;
+                struct list_head active_reqs;   /* used for cancellation */
+        } ____cacheline_aligned_in_smp;
+
         struct {
                 struct mutex    ring_lock;
-        } ____cacheline_aligned;
+                wait_queue_head_t wait;
+        } ____cacheline_aligned_in_smp;
 
         struct {
                 unsigned        tail;
                 spinlock_t      completion_lock;
-        } ____cacheline_aligned;
+        } ____cacheline_aligned_in_smp;
 
         struct page             *internal_pages[AIO_RING_PAGES];
-
-        struct rcu_head         rcu_head;
-        struct work_struct      rcu_work;
 };
 
 /*------ sysctl variables----*/