1 files changed, 55 insertions, 0 deletions

diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index a345cadad4dd..f9f13a32ddb8 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -364,6 +364,56 @@ rdma_rcl_chunk_count(struct rpcrdma_read_chunk *ch)
         return count;
 }
 
+/* If there was additional inline content, append it to the end of arg.pages.
+ * Tail copy has to be done after the reader function has determined how many
+ * pages are needed for RDMA READ.
+ */
+static int
+rdma_copy_tail(struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head,
+               u32 position, u32 byte_count, u32 page_offset, int page_no)
+{
+        char *srcp, *destp;
+        int ret;
+
+        ret = 0;
+        srcp = head->arg.head[0].iov_base + position;
+        byte_count = head->arg.head[0].iov_len - position;
+        if (byte_count > PAGE_SIZE) {
+                dprintk("svcrdma: large tail unsupported\n");
+                return 0;
+        }
+
+        /* Fit as much of the tail on the current page as possible */
+        if (page_offset != PAGE_SIZE) {
+                destp = page_address(rqstp->rq_arg.pages[page_no]);
+                destp += page_offset;
+                while (byte_count--) {
+                        *destp++ = *srcp++;
+                        page_offset++;
+                        if (page_offset == PAGE_SIZE && byte_count)
+                                goto more;
+                }
+                goto done;
+        }
+
+more:
+        /* Fit the rest on the next page */
+        page_no++;
+        destp = page_address(rqstp->rq_arg.pages[page_no]);
+        while (byte_count--)
+                *destp++ = *srcp++;
+
+        rqstp->rq_respages = &rqstp->rq_arg.pages[page_no+1];
+        rqstp->rq_next_page = rqstp->rq_respages + 1;
+
+done:
+        byte_count = head->arg.head[0].iov_len - position;
+        head->arg.page_len += byte_count;
+        head->arg.len += byte_count;
+        head->arg.buflen += byte_count;
+        return 1;
+}
+
 static int rdma_read_chunks(struct svcxprt_rdma *xprt,
                             struct rpcrdma_msg *rmsgp,
                             struct svc_rqst *rqstp,
@@ -440,9 +490,14 @@ static int rdma_read_chunks(struct svcxprt_rdma *xprt,
                 head->arg.page_len += pad;
                 head->arg.len += pad;
                 head->arg.buflen += pad;
+                page_offset += pad;
         }
 
         ret = 1;
+        if (position && position < head->arg.head[0].iov_len)
+                ret = rdma_copy_tail(rqstp, head, position,
+                                     byte_count, page_offset, page_no);
+        head->arg.head[0].iov_len = position;
         head->position = position;
 
  err: