mmc: support packed write command for eMMC4.5 devices

This patch supports packed write command of eMMC4.5 devices.  Several
writes can be grouped in packed command and all data of the individual
commands can be sent in a single transfer on the bus. Large amounts of
data in one transfer rather than several data of small size are
effective for eMMC write internally.  As a result, packed command help
write throughput be improved.  The following tables show the results
of packed write.

Type A:
test     none |  packed
iozone   25.8 |  31
tiotest  27.6 |  31.2
lmdd     31.2 |  35.4

Type B:
test     none |  packed
iozone   44.1 |  51.1
tiotest  47.9 |  52.5
lmdd     51.6 |  59.2

Type C:
test     none |  packed
iozone   19.5 |  32
tiotest  19.9 |  34.5
lmdd     22.8 |  40.7

Signed-off-by: Seungwon Jeon <tgih.jun@samsung.com>
Reviewed-by: Maya Erez <merez@codeaurora.org>
Reviewed-by: Namjae Jeon <linkinjeon@gmail.com>
Signed-off-by: Chris Ball <cjb@laptop.org>

1 files changed, 93 insertions, 3 deletions

diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c
index 5e0971016ac5..fa4e44ee7961 100644
--- a/drivers/mmc/card/queue.c
+++ b/drivers/mmc/card/queue.c
@@ -362,6 +362,49 @@ void mmc_cleanup_queue(struct mmc_queue *mq)
 }
 EXPORT_SYMBOL(mmc_cleanup_queue);
 
+int mmc_packed_init(struct mmc_queue *mq, struct mmc_card *card)
+{
+        struct mmc_queue_req *mqrq_cur = &mq->mqrq[0];
+        struct mmc_queue_req *mqrq_prev = &mq->mqrq[1];
+        int ret = 0;
+
+
+        mqrq_cur->packed = kzalloc(sizeof(struct mmc_packed), GFP_KERNEL);
+        if (!mqrq_cur->packed) {
+                pr_warn("%s: unable to allocate packed cmd for mqrq_cur\n",
+                        mmc_card_name(card));
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        mqrq_prev->packed = kzalloc(sizeof(struct mmc_packed), GFP_KERNEL);
+        if (!mqrq_prev->packed) {
+                pr_warn("%s: unable to allocate packed cmd for mqrq_prev\n",
+                        mmc_card_name(card));
+                kfree(mqrq_cur->packed);
+                mqrq_cur->packed = NULL;
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        INIT_LIST_HEAD(&mqrq_cur->packed->list);
+        INIT_LIST_HEAD(&mqrq_prev->packed->list);
+
+out:
+        return ret;
+}
+
+void mmc_packed_clean(struct mmc_queue *mq)
+{
+        struct mmc_queue_req *mqrq_cur = &mq->mqrq[0];
+        struct mmc_queue_req *mqrq_prev = &mq->mqrq[1];
+
+        kfree(mqrq_cur->packed);
+        mqrq_cur->packed = NULL;
+        kfree(mqrq_prev->packed);
+        mqrq_prev->packed = NULL;
+}
+
 /**
  * mmc_queue_suspend - suspend a MMC request queue
  * @mq: MMC queue to suspend
@@ -406,6 +449,41 @@ void mmc_queue_resume(struct mmc_queue *mq)
         }
 }
 
+static unsigned int mmc_queue_packed_map_sg(struct mmc_queue *mq,
+                                            struct mmc_packed *packed,
+                                            struct scatterlist *sg,
+                                            enum mmc_packed_type cmd_type)
+{
+        struct scatterlist *__sg = sg;
+        unsigned int sg_len = 0;
+        struct request *req;
+
+        if (mmc_packed_wr(cmd_type)) {
+                unsigned int hdr_sz = mmc_large_sector(mq->card) ? 4096 : 512;
+                unsigned int max_seg_sz = queue_max_segment_size(mq->queue);
+                unsigned int len, remain, offset = 0;
+                u8 *buf = (u8 *)packed->cmd_hdr;
+
+                remain = hdr_sz;
+                do {
+                        len = min(remain, max_seg_sz);
+                        sg_set_buf(__sg, buf + offset, len);
+                        offset += len;
+                        remain -= len;
+                        (__sg++)->page_link &= ~0x02;
+                        sg_len++;
+                } while (remain);
+        }
+
+        list_for_each_entry(req, &packed->list, queuelist) {
+                sg_len += blk_rq_map_sg(mq->queue, req, __sg);
+                __sg = sg + (sg_len - 1);
+                (__sg++)->page_link &= ~0x02;
+        }
+        sg_mark_end(sg + (sg_len - 1));
+        return sg_len;
+}
+
 /*
  * Prepare the sg list(s) to be handed of to the host driver
  */
@@ -414,14 +492,26 @@ unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq)
         unsigned int sg_len;
         size_t buflen;
         struct scatterlist *sg;
+        enum mmc_packed_type cmd_type;
         int i;
 
-        if (!mqrq->bounce_buf)
-                return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
+        cmd_type = mqrq->cmd_type;
+
+        if (!mqrq->bounce_buf) {
+                if (mmc_packed_cmd(cmd_type))
+                        return mmc_queue_packed_map_sg(mq, mqrq->packed,
+                                                       mqrq->sg, cmd_type);
+                else
+                        return blk_rq_map_sg(mq->queue, mqrq->req, mqrq->sg);
+        }
 
         BUG_ON(!mqrq->bounce_sg);
 
-        sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
+        if (mmc_packed_cmd(cmd_type))
+                sg_len = mmc_queue_packed_map_sg(mq, mqrq->packed,
+                                                 mqrq->bounce_sg, cmd_type);
+        else
+                sg_len = blk_rq_map_sg(mq->queue, mqrq->req, mqrq->bounce_sg);
 
         mqrq->bounce_sg_len = sg_len;