3 files changed, 137 insertions, 88 deletions

diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index a39645a54d82..262ba1f8ee50 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -222,12 +222,9 @@ enum iser_data_dir {
  * @size:         num entries of this sg
  * @data_len:     total beffer byte len
  * @dma_nents:    returned by dma_map_sg
- * @copy_buf:     allocated copy buf for SGs unaligned
- *                for rdma which are copied
  * @orig_sg:      pointer to the original sg list (in case
  *                we used a copy)
- * @sg_single:    SG-ified clone of a non SG SC or
- *                unaligned SG
+ * @orig_size:    num entris of orig sg list
  */
 struct iser_data_buf {
         struct scatterlist *sg;
@@ -235,8 +232,7 @@ struct iser_data_buf {
         unsigned long      data_len;
         unsigned int       dma_nents;
         struct scatterlist *orig_sg;
-        char               *copy_buf;
-        struct scatterlist sg_single;
+        unsigned int       orig_size;
   };
 
 /* fwd declarations */
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index b2e3b77340b5..3e2118e8ed87 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -674,28 +674,28 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
         /* if we were reading, copy back to unaligned sglist,
          * anyway dma_unmap and free the copy
          */
-        if (iser_task->data[ISER_DIR_IN].copy_buf) {
+        if (iser_task->data[ISER_DIR_IN].orig_sg) {
                 is_rdma_data_aligned = 0;
                 iser_finalize_rdma_unaligned_sg(iser_task,
                                                 &iser_task->data[ISER_DIR_IN],
                                                 ISER_DIR_IN);
         }
 
-        if (iser_task->data[ISER_DIR_OUT].copy_buf) {
+        if (iser_task->data[ISER_DIR_OUT].orig_sg) {
                 is_rdma_data_aligned = 0;
                 iser_finalize_rdma_unaligned_sg(iser_task,
                                                 &iser_task->data[ISER_DIR_OUT],
                                                 ISER_DIR_OUT);
         }
 
-        if (iser_task->prot[ISER_DIR_IN].copy_buf) {
+        if (iser_task->prot[ISER_DIR_IN].orig_sg) {
                 is_rdma_prot_aligned = 0;
                 iser_finalize_rdma_unaligned_sg(iser_task,
                                                 &iser_task->prot[ISER_DIR_IN],
                                                 ISER_DIR_IN);
         }
 
-        if (iser_task->prot[ISER_DIR_OUT].copy_buf) {
+        if (iser_task->prot[ISER_DIR_OUT].orig_sg) {
                 is_rdma_prot_aligned = 0;
                 iser_finalize_rdma_unaligned_sg(iser_task,
                                                 &iser_task->prot[ISER_DIR_OUT],
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index abc979e62ae1..f0cdc961eb11 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -39,7 +39,112 @@
 
 #include "iscsi_iser.h"
 
-#define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
+static void
+iser_free_bounce_sg(struct iser_data_buf *data)
+{
+        struct scatterlist *sg;
+        int count;
+
+        for_each_sg(data->sg, sg, data->size, count)
+                __free_page(sg_page(sg));
+
+        kfree(data->sg);
+
+        data->sg = data->orig_sg;
+        data->size = data->orig_size;
+        data->orig_sg = NULL;
+        data->orig_size = 0;
+}
+
+static int
+iser_alloc_bounce_sg(struct iser_data_buf *data)
+{
+        struct scatterlist *sg;
+        struct page *page;
+        unsigned long length = data->data_len;
+        int i = 0, nents = DIV_ROUND_UP(length, PAGE_SIZE);
+
+        sg = kcalloc(nents, sizeof(*sg), GFP_ATOMIC);
+        if (!sg)
+                goto err;
+
+        sg_init_table(sg, nents);
+        while (length) {
+                u32 page_len = min_t(u32, length, PAGE_SIZE);
+
+                page = alloc_page(GFP_ATOMIC);
+                if (!page)
+                        goto err;
+
+                sg_set_page(&sg[i], page, page_len, 0);
+                length -= page_len;
+                i++;
+        }
+
+        data->orig_sg = data->sg;
+        data->orig_size = data->size;
+        data->sg = sg;
+        data->size = nents;
+
+        return 0;
+
+err:
+        for (; i > 0; i--)
+                __free_page(sg_page(&sg[i - 1]));
+        kfree(sg);
+
+        return -ENOMEM;
+}
+
+static void
+iser_copy_bounce(struct iser_data_buf *data, bool to_buffer)
+{
+        struct scatterlist *osg, *bsg = data->sg;
+        void *oaddr, *baddr;
+        unsigned int left = data->data_len;
+        unsigned int bsg_off = 0;
+        int i;
+
+        for_each_sg(data->orig_sg, osg, data->orig_size, i) {
+                unsigned int copy_len, osg_off = 0;
+
+                oaddr = kmap_atomic(sg_page(osg)) + osg->offset;
+                copy_len = min(left, osg->length);
+                while (copy_len) {
+                        unsigned int len = min(copy_len, bsg->length - bsg_off);
+
+                        baddr = kmap_atomic(sg_page(bsg)) + bsg->offset;
+                        if (to_buffer)
+                                memcpy(baddr + bsg_off, oaddr + osg_off, len);
+                        else
+                                memcpy(oaddr + osg_off, baddr + bsg_off, len);
+
+                        kunmap_atomic(baddr - bsg->offset);
+                        osg_off += len;
+                        bsg_off += len;
+                        copy_len -= len;
+
+                        if (bsg_off >= bsg->length) {
+                                bsg = sg_next(bsg);
+                                bsg_off = 0;
+                        }
+                }
+                kunmap_atomic(oaddr - osg->offset);
+                left -= osg_off;
+        }
+}
+
+static inline void
+iser_copy_from_bounce(struct iser_data_buf *data)
+{
+        iser_copy_bounce(data, false);
+}
+
+static inline void
+iser_copy_to_bounce(struct iser_data_buf *data)
+{
+        iser_copy_bounce(data, true);
+}
 
 struct fast_reg_descriptor *
 iser_reg_desc_get(struct ib_conn *ib_conn)
@@ -75,52 +180,32 @@ static int iser_start_rdma_unali