Merge branch 'odp_fixes' into hmm.git

From rdma.git

Jason Gunthorpe says:

====================
This is a collection of general cleanups for ODP to clarify some of the
flows around umem creation and use of the interval tree.
====================

The branch is based on v5.3-rc5 due to dependencies, and is being taken
into hmm.git due to dependencies in the next patches.

* odp_fixes:
  RDMA/mlx5: Use odp instead of mr->umem in pagefault_mr
  RDMA/mlx5: Use ib_umem_start instead of umem.address
  RDMA/core: Make invalidate_range a device operation
  RDMA/odp: Use kvcalloc for the dma_list and page_list
  RDMA/odp: Check for overflow when computing the umem_odp end
  RDMA/odp: Provide ib_umem_odp_release() to undo the allocs
  RDMA/odp: Split creating a umem_odp from ib_umem_get
  RDMA/odp: Make the three ways to create a umem_odp clear
  RMDA/odp: Consolidate umem_odp initialization
  RDMA/odp: Make it clearer when a umem is an implicit ODP umem
  RDMA/odp: Iterate over the whole rbtree directly
  RDMA/odp: Use the common interval tree library instead of generic
  RDMA/mlx5: Fix MR npages calculation for IB_ACCESS_HUGETLB

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

1 files changed, 72 insertions, 24 deletions

diff --git a/fs/io_uring.c b/fs/io_uring.c
index e2a66e12fbc6..24bbe3cb7ad4 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -202,7 +202,7 @@ struct async_list {
 
         struct file             *file;
         off_t                   io_end;
-        size_t                  io_pages;
+        size_t                  io_len;
 };
 
 struct io_ring_ctx {
@@ -333,7 +333,8 @@ struct io_kiocb {
 #define REQ_F_IO_DRAIN          16      /* drain existing IO first */
 #define REQ_F_IO_DRAINED        32      /* drain done */
 #define REQ_F_LINK              64      /* linked sqes */
-#define REQ_F_FAIL_LINK         128     /* fail rest of links */
+#define REQ_F_LINK_DONE         128     /* linked sqes done */
+#define REQ_F_FAIL_LINK         256     /* fail rest of links */
         u64                     user_data;
         u32                     result;
         u32                     sequence;
@@ -429,7 +430,7 @@ static inline bool io_sequence_defer(struct io_ring_ctx *ctx,
         if ((req->flags & (REQ_F_IO_DRAIN|REQ_F_IO_DRAINED)) != REQ_F_IO_DRAIN)
                 return false;
 
-        return req->sequence > ctx->cached_cq_tail + ctx->sq_ring->dropped;
+        return req->sequence != ctx->cached_cq_tail + ctx->sq_ring->dropped;
 }
 
 static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx)
@@ -632,6 +633,7 @@ static void io_req_link_next(struct io_kiocb *req)
                         nxt->flags |= REQ_F_LINK;
                 }
 
+                nxt->flags |= REQ_F_LINK_DONE;
                 INIT_WORK(&nxt->work, io_sq_wq_submit_work);
                 queue_work(req->ctx->sqo_wq, &nxt->work);
         }
@@ -1064,8 +1066,42 @@ static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
          */
         offset = buf_addr - imu->ubuf;
         iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
-        if (offset)
-                iov_iter_advance(iter, offset);
+
+        if (offset) {
+                /*
+                 * Don't use iov_iter_advance() here, as it's really slow for
+                 * using the latter parts of a big fixed buffer - it iterates
+                 * over each segment manually. We can cheat a bit here, because
+                 * we know that:
+                 *
+                 * 1) it's a BVEC iter, we set it up
+                 * 2) all bvecs are PAGE_SIZE in size, except potentially the
+                 *    first and last bvec
+                 *
+                 * So just find our index, and adjust the iterator afterwards.
+                 * If the offset is within the first bvec (or the whole first
+                 * bvec, just use iov_iter_advance(). This makes it easier
+                 * since we can just skip the first segment, which may not
+                 * be PAGE_SIZE aligned.
+                 */
+                const struct bio_vec *bvec = imu->bvec;
+
+                if (offset <= bvec->bv_len) {
+                        iov_iter_advance(iter, offset);
+                } else {
+                        unsigned long seg_skip;
+
+                        /* skip first vec */
+                        offset -= bvec->bv_len;
+                        seg_skip = 1 + (offset >> PAGE_SHIFT);
+
+                        iter->bvec = bvec + seg_skip;
+                        iter->nr_segs -= seg_skip;
+                        iter->count -= bvec->bv_len + offset;
+                        iter->iov_offset = offset & ~PAGE_MASK;
+                }
+        }
+
         return 0;
 }
 
@@ -1120,28 +1156,26 @@ static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
         off_t io_end = kiocb->ki_pos + len;
 
         if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) {
-                unsigned long max_pages;
+                unsigned long max_bytes;
 
                 /* Use 8x RA size as a decent limiter for both reads/writes */
-                max_pages = filp->f_ra.ra_pages;
-                if (!max_pages)
-                        max_pages = VM_READAHEAD_PAGES;
-                max_pages *= 8;
-
-                /* If max pages are exceeded, reset the state */
-                len >>= PAGE_SHIFT;
-                if (async_list->io_pages + len <= max_pages) {
+                max_bytes = filp->f_ra.ra_pages << (PAGE_SHIFT + 3);
+                if (!max_bytes)
+                        max_bytes = VM_READAHEAD_PAGES << (PAGE_SHIFT + 3);
+
+                /* If max len are exceeded, reset the state */
+                if (async_list->io_len + len <= max_bytes) {
                         req->flags |= REQ_F_SEQ_PREV;
-                        async_list->io_pages += len;
+                        async_list->io_len += len;
                 } else {
                         io_end = 0;
-                        async_list->io_pages = 0;
+                        async_list->io_len = 0;
                 }
         }
 
         /* New file? Reset state. */
         if (async_list->file != filp) {
-                async_list->io_pages = 0;
+                async_list->io_len = 0;
                 async_list->file = filp;
         }
         async_list->io_end = io_end;
@@ -1630,6 +1664,8 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
         INIT_LIST_HEAD(&poll->wait.entry);
         init_waitqueue_func_entry(&poll->wait, io_poll_wake);
 
+        INIT_LIST_HEAD(&req->list);
+
         mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
 
         spin_lock_irq(&ctx->completion_lock);
@@ -1800,6 +1836,7 @@ restart:
         do {
                 struct sqe_submit *s = &req->submit;
                 const struct io_uring_sqe *sqe = s->sqe;
+                unsigned int flags = req->flags;
 
                 /* Ensure we clear previously set non-block flag */
                 req->rw.ki_flags &= ~IOCB_NOWAIT;
@@ -1844,6 +1881,10 @@ restart:
                 /* async context always use a copy of the sqe */
                 kfree(sqe);
 
+                /* req from defer and link list needn't decrease async cnt */
+                if (flags & (REQ_F_IO_DRAINED | REQ_F_LINK_DONE))
+                        goto out;
+
                 if (!async_list)
                         break;
                 if (!list_empty(&req_list)) {
@@ -1891,6 +1932,7 @@ restart:
                 }
         }
 
+out:
         if (cur_mm) {
                 set_fs(old_fs);
                 unuse_mm(cur_mm);
@@ -1917,6 +1959,10 @@ static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
         ret = true;
         spin_lock(&list->lock);
         list_add_tail(&req->list, &list->list);
+        /*
+         * Ensure we see a simultaneous modification from io_sq_wq_submit_work()
+         */
+        smp_mb();
         if (!atomic_read(&list->cnt)) {
                 list_del_init(&req->list);
                 ret = false;
@@ -1977,6 +2023,15 @@ static int io_queue_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 {
         int ret;
 
+        ret = io_req_defer(ctx, req, s->sqe);
+        if (ret) {
+                if (ret != -EIOCBQUEUED) {
+                        io_free_req(req);
+                        io_cqring_add_event(ctx, s->sqe->user_data, ret);
+                }
+                return 0;
+        }
+
         ret = __io_submit_sqe(ctx, req, s, true);
         if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
                 struct io_uring_sqe *sqe_copy;
@@ -2049,13 +2104,6 @@ err:
                 return;
         }
 
-        ret = io_req_defer(ctx, req, s->sqe);
-        if (ret) {
-                if (ret != -EIOCBQUEUED)
-                        goto err_req;
-                return;
-        }
-
         /*
          * If we already have a head request, queue this one for async
          * submittal once the head completes. If we don't have a head but