7 files changed, 81 insertions, 77 deletions

diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 175c9956cbe3..b64538b498dc 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -655,9 +655,10 @@ static void ffs_user_copy_worker(struct work_struct *work)
                 unuse_mm(io_data->mm);
         }
 
-        aio_complete(io_data->kiocb, ret, ret);
+        io_data->kiocb->ki_complete(io_data->kiocb, ret, ret);
 
-        if (io_data->ffs->ffs_eventfd && !io_data->kiocb->ki_eventfd)
+        if (io_data->ffs->ffs_eventfd &&
+            !(io_data->kiocb->ki_flags & IOCB_EVENTFD))
                 eventfd_signal(io_data->ffs->ffs_eventfd, 1);
 
         usb_ep_free_request(io_data->ep, io_data->req);
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index 200f9a584064..a4a80694f607 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -469,7 +469,7 @@ static void ep_user_copy_worker(struct work_struct *work)
                 ret = -EFAULT;
 
         /* completing the iocb can drop the ctx and mm, don't touch mm after */
-        aio_complete(iocb, ret, ret);
+        iocb->ki_complete(iocb, ret, ret);
 
         kfree(priv->buf);
         kfree(priv->to_free);
@@ -497,7 +497,8 @@ static void ep_aio_complete(struct usb_ep *ep, struct usb_request *req)
                 kfree(priv);
                 iocb->private = NULL;
                 /* aio_complete() reports bytes-transferred _and_ faults */
-                aio_complete(iocb, req->actual ? req->actual : req->status,
+
+                iocb->ki_complete(iocb, req->actual ? req->actual : req->status,
                                 req->status);
         } else {
                 /* ep_copy_to_user() won't report both; we hide some faults */
diff --git a/fs/aio.c b/fs/aio.c
index 8ca8df1c3550..958286536a10 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -151,6 +151,38 @@ struct kioctx {
         unsigned                id;
 };
 
+/*
+ * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
+ * cancelled or completed (this makes a certain amount of sense because
+ * successful cancellation - io_cancel() - does deliver the completion to
+ * userspace).
+ *
+ * And since most things don't implement kiocb cancellation and we'd really like
+ * kiocb completion to be lockless when possible, we use ki_cancel to
+ * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
+ * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
+ */
+#define KIOCB_CANCELLED         ((void *) (~0ULL))
+
+struct aio_kiocb {
+        struct kiocb            common;
+
+        struct kioctx           *ki_ctx;
+        kiocb_cancel_fn         *ki_cancel;
+
+        struct iocb __user      *ki_user_iocb;  /* user's aiocb */
+        __u64                   ki_user_data;   /* user's data for completion */
+
+        struct list_head        ki_list;        /* the aio core uses this
+                                                 * for cancellation */
+
+        /*
+         * If the aio_resfd field of the userspace iocb is not zero,
+         * this is the underlying eventfd context to deliver events to.
+         */
+        struct eventfd_ctx      *ki_eventfd;
+};
+
 /*------ sysctl variables----*/
 static DEFINE_SPINLOCK(aio_nr_lock);
 unsigned long aio_nr;           /* current system wide number of aio requests */
@@ -220,7 +252,7 @@ static int __init aio_setup(void)
         if (IS_ERR(aio_mnt))
                 panic("Failed to create aio fs mount.");
 
-        kiocb_cachep = KMEM_CACHE(kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
+        kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
         kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
 
         pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
@@ -480,8 +512,9 @@ static int aio_setup_ring(struct kioctx *ctx)
 #define AIO_EVENTS_FIRST_PAGE   ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
 #define AIO_EVENTS_OFFSET       (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
 
-void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel)
+void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
 {
+        struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, common);
         struct kioctx *ctx = req->ki_ctx;
         unsigned long flags;
 
@@ -496,7 +529,7 @@ void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel)
 }
 EXPORT_SYMBOL(kiocb_set_cancel_fn);
 
-static int kiocb_cancel(struct kiocb *kiocb)
+static int kiocb_cancel(struct aio_kiocb *kiocb)
 {
         kiocb_cancel_fn *old, *cancel;
 
@@ -514,7 +547,7 @@ static int kiocb_cancel(struct kiocb *kiocb)
                 cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
         } while (cancel != old);
 
-        return cancel(kiocb);
+        return cancel(&kiocb->common);
 }
 
 static void free_ioctx(struct work_struct *work)
@@ -550,13 +583,13 @@ static void free_ioctx_reqs(struct percpu_ref *ref)
 static void free_ioctx_users(struct percpu_ref *ref)
 {
         struct kioctx *ctx = container_of(ref, struct kioctx, users);
-        struct kiocb *req;
+        struct aio_kiocb *req;
 
         spin_lock_irq(&ctx->ctx_lock);
 
         while (!list_empty(&ctx->active_reqs)) {
                 req = list_first_entry(&ctx->active_reqs,
-                                       struct kiocb, ki_list);
+                                       struct aio_kiocb, ki_list);
 
                 list_del_init(&req->ki_list);
                 kiocb_cancel(req);
@@ -932,9 +965,9 @@ static void user_refill_reqs_available(struct kioctx *ctx)
  *      Allocate a slot for an aio request.
  * Returns NULL if no requests are free.
  */
-static inline struct kiocb *aio_get_req(struct kioctx *ctx)
+static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
 {
-        struct kiocb *req;
+        struct aio_kiocb *req;
 
         if (!get_reqs_available(ctx)) {
                 user_refill_reqs_available(ctx);
@@ -955,10 +988,10 @@ out_put:
         return NULL;
 }
 
-static void kiocb_free(struct kiocb *req)
+static void kiocb_free(struct aio_kiocb *req)
 {
-        if (req->ki_filp)
-                fput(req->ki_filp);
+        if (req->common.ki_filp)
+                fput(req->common.ki_filp);
         if (req->ki_eventfd != NULL)
                 eventfd_ctx_put(req->ki_eventfd);
         kmem_cache_free(kiocb_cachep, req);
@@ -994,8 +1027,9 @@ out:
 /* aio_complete
  *      Called when the io request on the given iocb is complete.
  */
-void aio_complete(struct kiocb *iocb, long res, long res2)
+static void aio_complete(struct kiocb *kiocb, long res, long res2)
 {
+        struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, common);
         struct kioctx   *ctx = iocb->ki_ctx;
         struct aio_ring *ring;
         struct io_event *ev_page, *event;
@@ -1009,7 +1043,7 @@ void aio_complete(struct kiocb *iocb, long res, long res2)