fs: split generic and aio kiocb

Most callers in the kernel want to perform synchronous file I/O, but
still have to bloat the stack with a full struct kiocb.  Split out
the parts needed in filesystem code from those in the aio code, and
only allocate those needed to pass down argument on the stack.  The
aio code embedds the generic iocb in the one it allocates and can
easily get back to it by using container_of.

Also add a ->ki_complete method to struct kiocb, this is used to call
into the aio code and thus removes the dependency on aio for filesystems
impementing asynchronous operations.  It will also allow other callers
to substitute their own completion callback.

We also add a new ->ki_flags field to work around the nasty layering
violation recently introduced in commit 5e33f6 ("usb: gadget: ffs: add
eventfd notification about ffs events").

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

7 files changed, 81 insertions, 77 deletions

diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 175c9956cbe3..b64538b498dc 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -655,9 +655,10 @@ static void ffs_user_copy_worker(struct work_struct *work)
                 unuse_mm(io_data->mm);
         }
 
-        aio_complete(io_data->kiocb, ret, ret);
+        io_data->kiocb->ki_complete(io_data->kiocb, ret, ret);
 
-        if (io_data->ffs->ffs_eventfd && !io_data->kiocb->ki_eventfd)
+        if (io_data->ffs->ffs_eventfd &&
+            !(io_data->kiocb->ki_flags & IOCB_EVENTFD))
                 eventfd_signal(io_data->ffs->ffs_eventfd, 1);
 
         usb_ep_free_request(io_data->ep, io_data->req);
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index 200f9a584064..a4a80694f607 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -469,7 +469,7 @@ static void ep_user_copy_worker(struct work_struct *work)
                 ret = -EFAULT;
 
         /* completing the iocb can drop the ctx and mm, don't touch mm after */
-        aio_complete(iocb, ret, ret);
+        iocb->ki_complete(iocb, ret, ret);
 
         kfree(priv->buf);
         kfree(priv->to_free);
@@ -497,7 +497,8 @@ static void ep_aio_complete(struct usb_ep *ep, struct usb_request *req)
                 kfree(priv);
                 iocb->private = NULL;
                 /* aio_complete() reports bytes-transferred _and_ faults */
-                aio_complete(iocb, req->actual ? req->actual : req->status,
+
+                iocb->ki_complete(iocb, req->actual ? req->actual : req->status,
                                 req->status);
         } else {
                 /* ep_copy_to_user() won't report both; we hide some faults */
diff --git a/fs/aio.c b/fs/aio.c
index 8ca8df1c3550..958286536a10 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -151,6 +151,38 @@ struct kioctx {
         unsigned                id;
 };
 
+/*
+ * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
+ * cancelled or completed (this makes a certain amount of sense because
+ * successful cancellation - io_cancel() - does deliver the completion to
+ * userspace).
+ *
+ * And since most things don't implement kiocb cancellation and we'd really like
+ * kiocb completion to be lockless when possible, we use ki_cancel to
+ * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
+ * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
+ */
+#define KIOCB_CANCELLED         ((void *) (~0ULL))
+
+struct aio_kiocb {
+        struct kiocb            common;
+
+        struct kioctx           *ki_ctx;
+        kiocb_cancel_fn         *ki_cancel;
+
+        struct iocb __user      *ki_user_iocb;  /* user's aiocb */
+        __u64                   ki_user_data;   /* user's data for completion */
+
+        struct list_head        ki_list;        /* the aio core uses this
+                                                 * for cancellation */
+
+        /*
+         * If the aio_resfd field of the userspace iocb is not zero,
+         * this is the underlying eventfd context to deliver events to.
+         */
+        struct eventfd_ctx      *ki_eventfd;
+};
+
 /*------ sysctl variables----*/
 static DEFINE_SPINLOCK(aio_nr_lock);
 unsigned long aio_nr;           /* current system wide number of aio requests */
@@ -220,7 +252,7 @@ static int __init aio_setup(void)
         if (IS_ERR(aio_mnt))
                 panic("Failed to create aio fs mount.");
 
-        kiocb_cachep = KMEM_CACHE(kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
+        kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
         kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
 
         pr_debug("sizeof(struct page) = %zu\n", sizeof(struct page));
@@ -480,8 +512,9 @@ static int aio_setup_ring(struct kioctx *ctx)
 #define AIO_EVENTS_FIRST_PAGE   ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
 #define AIO_EVENTS_OFFSET       (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
 
-void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel)
+void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel)
 {
+        struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, common);
         struct kioctx *ctx = req->ki_ctx;
         unsigned long flags;
 
@@ -496,7 +529,7 @@ void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel)
 }
 EXPORT_SYMBOL(kiocb_set_cancel_fn);
 
-static int kiocb_cancel(struct kiocb *kiocb)
+static int kiocb_cancel(struct aio_kiocb *kiocb)
 {
         kiocb_cancel_fn *old, *cancel;
 
@@ -514,7 +547,7 @@ static int kiocb_cancel(struct kiocb *kiocb)
                 cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
         } while (cancel != old);
 
-        return cancel(kiocb);
+        return cancel(&kiocb->common);
 }
 
 static void free_ioctx(struct work_struct *work)
@@ -550,13 +583,13 @@ static void free_ioctx_reqs(struct percpu_ref *ref)
 static void free_ioctx_users(struct percpu_ref *ref)
 {
         struct kioctx *ctx = container_of(ref, struct kioctx, users);
-        struct kiocb *req;
+        struct aio_kiocb *req;
 
         spin_lock_irq(&ctx->ctx_lock);
 
         while (!list_empty(&ctx->active_reqs)) {
                 req = list_first_entry(&ctx->active_reqs,
-                                       struct kiocb, ki_list);
+                                       struct aio_kiocb, ki_list);
 
                 list_del_init(&req->ki_list);
                 kiocb_cancel(req);
@@ -932,9 +965,9 @@ static void user_refill_reqs_available(struct kioctx *ctx)
  *      Allocate a slot for an aio request.
  * Returns NULL if no requests are free.
  */
-static inline struct kiocb *aio_get_req(struct kioctx *ctx)
+static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx)
 {
-        struct kiocb *req;
+        struct aio_kiocb *req;
 
         if (!get_reqs_available(ctx)) {
                 user_refill_reqs_available(ctx);
@@ -955,10 +988,10 @@ out_put:
         return NULL;
 }
 
-static void kiocb_free(struct kiocb *req)
+static void kiocb_free(struct aio_kiocb *req)
 {
-        if (req->ki_filp)
-                fput(req->ki_filp);
+        if (req->common.ki_filp)
+                fput(req->common.ki_filp);
         if (req->ki_eventfd != NULL)
                 eventfd_ctx_put(req->ki_eventfd);
         kmem_cache_free(kiocb_cachep, req);
@@ -994,8 +1027,9 @@ out:
 /* aio_complete
  *      Called when the io request on the given iocb is complete.
  */
-void aio_complete(struct kiocb *iocb, long res, long res2)
+stat