1 files changed, 65 insertions, 17 deletions

diff --git a/fs/io_uring.c b/fs/io_uring.c
index e2a66e12fbc6..d542f1cf4428 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -202,7 +202,7 @@ struct async_list {
 
         struct file             *file;
         off_t                   io_end;
-        size_t                  io_pages;
+        size_t                  io_len;
 };
 
 struct io_ring_ctx {
@@ -333,7 +333,8 @@ struct io_kiocb {
 #define REQ_F_IO_DRAIN          16      /* drain existing IO first */
 #define REQ_F_IO_DRAINED        32      /* drain done */
 #define REQ_F_LINK              64      /* linked sqes */
-#define REQ_F_FAIL_LINK         128     /* fail rest of links */
+#define REQ_F_LINK_DONE         128     /* linked sqes done */
+#define REQ_F_FAIL_LINK         256     /* fail rest of links */
         u64                     user_data;
         u32                     result;
         u32                     sequence;
@@ -429,7 +430,7 @@ static inline bool io_sequence_defer(struct io_ring_ctx *ctx,
         if ((req->flags & (REQ_F_IO_DRAIN|REQ_F_IO_DRAINED)) != REQ_F_IO_DRAIN)
                 return false;
 
-        return req->sequence > ctx->cached_cq_tail + ctx->sq_ring->dropped;
+        return req->sequence != ctx->cached_cq_tail + ctx->sq_ring->dropped;
 }
 
 static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
@@ -632,6 +633,7 @@ static void io_req_link_next(struct io_kiocb *req)
                         nxt->flags |= REQ_F_LINK;
                 }
 
+                nxt->flags |= REQ_F_LINK_DONE;
                 INIT_WORK(&nxt->work, io_sq_wq_submit_work);
                 queue_work(req->ctx->sqo_wq, &nxt->work);
         }
@@ -1064,8 +1066,44 @@ static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
          */
         offset = buf_addr - imu->ubuf;
         iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
-        if (offset)
-                iov_iter_advance(iter, offset);
+
+        if (offset) {
+                /*
+                 * Don't use iov_iter_advance() here, as it's really slow for
+                 * using the latter parts of a big fixed buffer - it iterates
+                 * over each segment manually. We can cheat a bit here, because
+                 * we know that:
+                 *
+                 * 1) it's a BVEC iter, we set it up
+                 * 2) all bvecs are PAGE_SIZE in size, except potentially the
+                 *    first and last bvec
+                 *
+                 * So just find our index, and adjust the iterator afterwards.
+                 * If the offset is within the first bvec (or the whole first
+                 * bvec, just use iov_iter_advance(). This makes it easier
+                 * since we can just skip the first segment, which may not
+                 * be PAGE_SIZE aligned.
+                 */
+                const struct bio_vec *bvec = imu->bvec;
+
+                if (offset <= bvec->bv_len) {
+                        iov_iter_advance(iter, offset);
+                } else {
+                        unsigned long seg_skip;
+
+                        /* skip first vec */
+                        offset -= bvec->bv_len;
+                        seg_skip = 1 + (offset >> PAGE_SHIFT);
+
+                        iter->bvec = bvec + seg_skip;
+                        iter->nr_segs -= seg_skip;
+                        iter->count -= (seg_skip << PAGE_SHIFT);
+                        iter->iov_offset = offset & ~PAGE_MASK;
+                        if (iter->iov_offset)
+                                iter->count -= iter->iov_offset;
+                }
+        }
+
         return 0;
 }
 
@@ -1120,28 +1158,26 @@ static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
         off_t io_end = kiocb->ki_pos + len;
 
         if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) {
-                unsigned long max_pages;
+                unsigned long max_bytes;
 
                 /* Use 8x RA size as a decent limiter for both reads/writes */
-                max_pages = filp->f_ra.ra_pages;
-                if (!max_pages)
-                        max_pages = VM_READAHEAD_PAGES;
-                max_pages *= 8;
-
-                /* If max pages are exceeded, reset the state */
-                len >>= PAGE_SHIFT;
-                if (async_list->io_pages + len <= max_pages) {
+                max_bytes = filp->f_ra.ra_pages << (PAGE_SHIFT + 3);
+                if (!max_bytes)
+                        max_bytes = VM_READAHEAD_PAGES << (PAGE_SHIFT + 3);
+
+                /* If max len are exceeded, reset the state */
+                if (async_list->io_len + len <= max_bytes) {
                         req->flags |= REQ_F_SEQ_PREV;
-                        async_list->io_pages += len;
+                        async_list->io_len += len;
                 } else {
                         io_end = 0;
-                        async_list->io_pages = 0;
+                        async_list->io_len = 0;
                 }
         }
 
         /* New file? Reset state. */
         if (async_list->file != filp) {
-                async_list->io_pages = 0;
+                async_list->io_len = 0;
                 async_list->file = filp;
         }
         async_list->io_end = io_end;
@@ -1630,6 +1666,8 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
         INIT_LIST_HEAD(&poll->wait.entry);
         init_waitqueue_func_entry(&poll->wait, io_poll_wake);
 
+        INIT_LIST_HEAD(&req->list);
+
         mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
 
         spin_lock_irq(&ctx->completion_lock);
@@ -1800,6 +1838,7 @@ restart:
         do {
                 struct sqe_submit *s = &req->submit;
                 const struct io_uring_sqe *sqe = s->sqe;
+                unsigned int flags = req->flags;
 
                 /* Ensure we clear previously set non-block flag */
                 req->rw.ki_flags &= ~IOCB_NOWAIT;
@@ -1844,6 +1883,10 @@ restart:
                 /* async context always use a copy of the sqe */
                 kfree(sqe);
 
+                /* req from defer and link list needn't decrease async cnt */
+                if (flags & (REQ_F_IO_DRAINED | REQ_F_LINK_DONE))
+                        goto out;
+
                 if (!async_list)
                         break;
                 if (!list_empty(&req_list)) {
@@ -1891,6 +1934,7 @@ restart:
                 }
         }
 
+out:
         if (cur_mm) {
                 set_fs(old_fs);
                 unuse_mm(cur_mm);
@@ -1917,6 +1961,10 @@ static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
         ret = true;
         spin_lock(&list->lock);
         list_add_tail(&req->list, &list->list);
+        /*
+         * Ensure we see a simultaneous modification from io_sq_wq_submit_work()
+         */
+        smp_mb();
         if (!atomic_read(&list->cnt)) {
                 list_del_init(&req->list);
                 ret = false;