aio: make the lookup_ioctx() lockless

The mm->ioctx_list is currently protected by a reader-writer lock,
so we always grab that lock on the read side for doing ioctx
lookups. As the workload is extremely reader biased, turn this into
an rcu hlist so we can make lookup_ioctx() lockless. Get rid of
the rwlock and use a spinlock for providing update side exclusion.

There's usually only 1 entry on this list, so it doesn't make sense
to look into fancier data structures.

Reviewed-by: Jeff Moyer <jmoyer@redhat.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

2 files changed, 7 insertions, 3 deletions

diff --git a/include/linux/aio.h b/include/linux/aio.h
index f6b8cf99b596..b16a957030f8 100644
--- a/include/linux/aio.h
+++ b/include/linux/aio.h
@@ -5,6 +5,7 @@
 #include <linux/workqueue.h>
 #include <linux/aio_abi.h>
 #include <linux/uio.h>
+#include <linux/rcupdate.h>
 
 #include <asm/atomic.h>
 
@@ -183,7 +184,7 @@ struct kioctx {
 
         /* This needs improving */
         unsigned long           user_id;
-        struct kioctx           *next;
+        struct hlist_node       list;
 
         wait_queue_head_t       wait;
 
@@ -199,6 +200,8 @@ struct kioctx {
         struct aio_ring_info    ring_info;
 
         struct delayed_work     wq;
+
+        struct rcu_head         rcu_head;
 };
 
 /* prototypes */
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index fe825471d5aa..9cfc9b627fdd 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -232,8 +232,9 @@ struct mm_struct {
         struct core_state *core_state; /* coredumping support */
 
         /* aio bits */
-        rwlock_t                ioctx_list_lock;        /* aio lock */
-        struct kioctx           *ioctx_list;
+        spinlock_t              ioctx_lock;
+        struct hlist_head       ioctx_list;
+
 #ifdef CONFIG_MM_OWNER
         /*
          * "owner" points to a task that is regarded as the canonical