1 files changed, 45 insertions, 53 deletions

diff --git a/fs/aio.c b/fs/aio.c
index 0e11e31dbb77..d6b1551342b7 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -741,19 +741,9 @@ static ssize_t aio_run_iocb(struct kiocb *iocb)
         ret = retry(iocb);
         current->io_wait = NULL;
 
-        if (-EIOCBRETRY != ret) {
-                if (-EIOCBQUEUED != ret) {
-                        BUG_ON(!list_empty(&iocb->ki_wait.task_list));
-                        aio_complete(iocb, ret, 0);
-                        /* must not access the iocb after this */
-                }
-        } else {
-                /*
-                 * Issue an additional retry to avoid waiting forever if
-                 * no waits were queued (e.g. in case of a short read).
-                 */
-                if (list_empty(&iocb->ki_wait.task_list))
-                        kiocbSetKicked(iocb);
+        if (ret != -EIOCBRETRY && ret != -EIOCBQUEUED) {
+                BUG_ON(!list_empty(&iocb->ki_wait.task_list));
+                aio_complete(iocb, ret, 0);
         }
 out:
         spin_lock_irq(&ctx->ctx_lock);
@@ -899,16 +889,24 @@ static void aio_kick_handler(void *data)
  * and if required activate the aio work queue to process
  * it
  */
-static void queue_kicked_iocb(struct kiocb *iocb)
+static void try_queue_kicked_iocb(struct kiocb *iocb)
 {
         struct kioctx   *ctx = iocb->ki_ctx;
         unsigned long flags;
         int run = 0;
 
-        WARN_ON((!list_empty(&iocb->ki_wait.task_list)));
+        /* We're supposed to be the only path putting the iocb back on the run
+         * list.  If we find that the iocb is *back* on a wait queue already
+         * than retry has happened before we could queue the iocb.  This also
+         * means that the retry could have completed and freed our iocb, no
+         * good. */
+        BUG_ON((!list_empty(&iocb->ki_wait.task_list)));
 
         spin_lock_irqsave(&ctx->ctx_lock, flags);
-        run = __queue_kicked_iocb(iocb);
+        /* set this inside the lock so that we can't race with aio_run_iocb()
+         * testing it and putting the iocb on the run list under the lock */
+        if (!kiocbTryKick(iocb))
+                run = __queue_kicked_iocb(iocb);
         spin_unlock_irqrestore(&ctx->ctx_lock, flags);
         if (run)
                 aio_queue_work(ctx);
@@ -931,10 +929,7 @@ void fastcall kick_iocb(struct kiocb *iocb)
                 return;
         }
 
-        /* If its already kicked we shouldn't queue it again */
-        if (!kiocbTryKick(iocb)) {
-                queue_kicked_iocb(iocb);
-        }
+        try_queue_kicked_iocb(iocb);
 }
 EXPORT_SYMBOL(kick_iocb);
 
@@ -1322,8 +1317,11 @@ asmlinkage long sys_io_destroy(aio_context_t ctx)
 }
 
 /*
- * Default retry method for aio_read (also used for first time submit)
- * Responsible for updating iocb state as retries progress
+ * aio_p{read,write} are the default  ki_retry methods for
+ * IO_CMD_P{READ,WRITE}.  They maintains kiocb retry state around potentially
+ * multiple calls to f_op->aio_read().  They loop around partial progress
+ * instead of returning -EIOCBRETRY because they don't have the means to call
+ * kick_iocb().
  */
 static ssize_t aio_pread(struct kiocb *iocb)
 {
@@ -1332,25 +1330,25 @@ static ssize_t aio_pread(struct kiocb *iocb)
         struct inode *inode = mapping->host;
         ssize_t ret = 0;
 
-        ret = file->f_op->aio_read(iocb, iocb->ki_buf,
-                iocb->ki_left, iocb->ki_pos);
+        do {
+                ret = file->f_op->aio_read(iocb, iocb->ki_buf,
+                        iocb->ki_left, iocb->ki_pos);
+                /*
+                 * Can't just depend on iocb->ki_left to determine
+                 * whether we are done. This may have been a short read.
+                 */
+                if (ret > 0) {
+                        iocb->ki_buf += ret;
+                        iocb->ki_left -= ret;
+                }
 
-        /*
-         * Can't just depend on iocb->ki_left to determine
-         * whether we are done. This may have been a short read.
-         */
-        if (ret > 0) {
-                iocb->ki_buf += ret;
-                iocb->ki_left -= ret;
                 /*
-                 * For pipes and sockets we return once we have
-                 * some data; for regular files we retry till we
-                 * complete the entire read or find that we can't
-                 * read any more data (e.g short reads).
+                 * For pipes and sockets we return once we have some data; for
+                 * regular files we retry till we complete the entire read or
+                 * find that we can't read any more data (e.g short reads).
                  */
-                if (!S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode))
-                        ret = -EIOCBRETRY;
-        }
+        } while (ret > 0 && iocb->ki_left > 0 &&
+                 !S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode));
 
         /* This means we must have transferred all that we could */
         /* No need to retry anymore */
@@ -1360,27 +1358,21 @@ static ssize_t aio_pread(struct kiocb *iocb)
         return ret;
 }
 
-/*
- * Default retry method for aio_write (also used for first time submit)
- * Responsible for updating iocb state as retries progress
- */
+/* see aio_pread() */
 static ssize_t aio_pwrite(struct kiocb *iocb)
 {
         struct file *file = iocb->ki_filp;
         ssize_t ret = 0;
 
-        ret = file->f_op->aio_write(iocb, iocb->ki_buf,
-                iocb->ki_left, iocb->ki_pos);
-
-        if (ret > 0) {
-                iocb->ki_buf += ret;
-                iocb->ki_left -= ret;
-
-                ret = -EIOCBRETRY;
-        }
+        do {
+                ret = file->f_op->aio_write(iocb, iocb->ki_buf,
+                        iocb->ki_left, iocb->ki_pos);
+                if (ret > 0) {
+                        iocb->ki_buf += ret;
+                        iocb->ki_left -= ret;
+                }
+        } while (ret > 0 && iocb->ki_left > 0);
 
-        /* This means we must have transferred all that we could */
-        /* No need to retry anymore */
         if ((ret == 0) || (iocb->ki_left == 0))
                 ret = iocb->ki_nbytes - iocb->ki_left;