mmc: block: Add CQE support

Add CQE support to the block driver, including:
    - optionally using DCMD for flush requests
    - "manually" issuing discard requests
    - issuing read / write requests to the CQE
    - supporting block-layer timeouts
    - handling recovery
    - supporting re-tuning

CQE offers 25% - 50% better random multi-threaded I/O.  There is a slight
(e.g. 2%) drop in sequential read speed but no observable change to sequential
write.

CQE automatically sends the commands to complete requests.  However it only
supports reads / writes and so-called "direct commands" (DCMD).  Furthermore
DCMD is limited to one command at a time, but discards require 3 commands.
That makes issuing discards through CQE very awkward, but some CQE's don't
support DCMD anyway.  So for discards, the existing non-CQE approach is
taken, where the mmc core code issues the 3 commands one at a time i.e.
mmc_erase(). Where DCMD is used, is for issuing flushes.

Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
Acked-by: Linus Walleij <linus.walleij@linaro.org>
Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
Tested-by: Linus Walleij <linus.walleij@linaro.org>

1 files changed, 18 insertions, 0 deletions

diff --git a/drivers/mmc/core/queue.h b/drivers/mmc/core/queue.h
index ce9249852f26..1d7d3b0afff8 100644
--- a/drivers/mmc/core/queue.h
+++ b/drivers/mmc/core/queue.h
@@ -17,6 +17,7 @@ enum mmc_issued {
 
 enum mmc_issue_type {
         MMC_ISSUE_SYNC,
+        MMC_ISSUE_DCMD,
         MMC_ISSUE_ASYNC,
         MMC_ISSUE_MAX,
 };
@@ -92,8 +93,15 @@ struct mmc_queue {
         int                     qcnt;
 
         int                     in_flight[MMC_ISSUE_MAX];
+        unsigned int            cqe_busy;
+#define MMC_CQE_DCMD_BUSY       BIT(0)
+#define MMC_CQE_QUEUE_FULL      BIT(1)
+        bool                    use_cqe;
+        bool                    recovery_needed;
+        bool                    in_recovery;
         bool                    rw_wait;
         bool                    waiting;
+        struct work_struct      recovery_work;
         wait_queue_head_t       wait;
         struct request          *complete_req;
         struct mutex            complete_lock;
@@ -108,11 +116,21 @@ extern void mmc_queue_resume(struct mmc_queue *);
 extern unsigned int mmc_queue_map_sg(struct mmc_queue *,
                                      struct mmc_queue_req *);
 
+void mmc_cqe_check_busy(struct mmc_queue *mq);
+void mmc_cqe_recovery_notifier(struct mmc_request *mrq);
+
 enum mmc_issue_type mmc_issue_type(struct mmc_queue *mq, struct request *req);
 
 static inline int mmc_tot_in_flight(struct mmc_queue *mq)
 {
         return mq->in_flight[MMC_ISSUE_SYNC] +
+               mq->in_flight[MMC_ISSUE_DCMD] +
+               mq->in_flight[MMC_ISSUE_ASYNC];
+}
+
+static inline int mmc_cqe_qcnt(struct mmc_queue *mq)
+{
+        return mq->in_flight[MMC_ISSUE_DCMD] +
                mq->in_flight[MMC_ISSUE_ASYNC];
 }