Merge tag 'for-4.21/block-20181221' of git://git.kernel.dk/linux-block

Pull block updates from Jens Axboe:
 "This is the main pull request for block/storage for 4.21.

  Larger than usual, it was a busy round with lots of goodies queued up.
  Most notable is the removal of the old IO stack, which has been a long
  time coming. No new features for a while, everything coming in this
  week has all been fixes for things that were previously merged.

  This contains:

   - Use atomic counters instead of semaphores for mtip32xx (Arnd)

   - Cleanup of the mtip32xx request setup (Christoph)

   - Fix for circular locking dependency in loop (Jan, Tetsuo)

   - bcache (Coly, Guoju, Shenghui)
      * Optimizations for writeback caching
      * Various fixes and improvements

   - nvme (Chaitanya, Christoph, Sagi, Jay, me, Keith)
      * host and target support for NVMe over TCP
      * Error log page support
      * Support for separate read/write/poll queues
      * Much improved polling
      * discard OOM fallback
      * Tracepoint improvements

   - lightnvm (Hans, Hua, Igor, Matias, Javier)
      * Igor added packed metadata to pblk. Now drives without metadata
        per LBA can be used as well.
      * Fix from Geert on uninitialized value on chunk metadata reads.
      * Fixes from Hans and Javier to pblk recovery and write path.
      * Fix from Hua Su to fix a race condition in the pblk recovery
        code.
      * Scan optimization added to pblk recovery from Zhoujie.
      * Small geometry cleanup from me.

   - Conversion of the last few drivers that used the legacy path to
     blk-mq (me)

   - Removal of legacy IO path in SCSI (me, Christoph)

   - Removal of legacy IO stack and schedulers (me)

   - Support for much better polling, now without interrupts at all.
     blk-mq adds support for multiple queue maps, which enables us to
     have a map per type. This in turn enables nvme to have separate
     completion queues for polling, which can then be interrupt-less.
     Also means we're ready for async polled IO, which is hopefully
     coming in the next release.

   - Killing of (now) unused block exports (Christoph)

   - Unification of the blk-rq-qos and blk-wbt wait handling (Josef)

   - Support for zoned testing with null_blk (Masato)

   - sx8 conversion to per-host tag sets (Christoph)

   - IO priority improvements (Damien)

   - mq-deadline zoned fix (Damien)

   - Ref count blkcg series (Dennis)

   - Lots of blk-mq improvements and speedups (me)

   - sbitmap scalability improvements (me)

   - Make core inflight IO accounting per-cpu (Mikulas)

   - Export timeout setting in sysfs (Weiping)

   - Cleanup the direct issue path (Jianchao)

   - Export blk-wbt internals in block debugfs for easier debugging
     (Ming)

   - Lots of other fixes and improvements"

* tag 'for-4.21/block-20181221' of git://git.kernel.dk/linux-block: (364 commits)
  kyber: use sbitmap add_wait_queue/list_del wait helpers
  sbitmap: add helpers for add/del wait queue handling
  block: save irq state in blkg_lookup_create()
  dm: don't reuse bio for flushes
  nvme-pci: trace SQ status on completions
  nvme-rdma: implement polling queue map
  nvme-fabrics: allow user to pass in nr_poll_queues
  nvme-fabrics: allow nvmf_connect_io_queue to poll
  nvme-core: optionally poll sync commands
  block: make request_to_qc_t public
  nvme-tcp: fix spelling mistake "attepmpt" -> "attempt"
  nvme-tcp: fix endianess annotations
  nvmet-tcp: fix endianess annotations
  nvme-pci: refactor nvme_poll_irqdisable to make sparse happy
  nvme-pci: only set nr_maps to 2 if poll queues are supported
  nvmet: use a macro for default error location
  nvmet: fix comparison of a u16 with -1
  blk-mq: enable IO poll if .nr_queues of type poll > 0
  blk-mq: change blk_mq_queue_busy() to blk_mq_queue_inflight()
  blk-mq: skip zero-queue maps in blk_mq_map_swqueue
  ...

12 files changed, 259 insertions, 209 deletions

diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c
index 8b2b72b93885..da58020a144e 100644
--- a/drivers/ide/ide-atapi.c
+++ b/drivers/ide/ide-atapi.c
@@ -94,7 +94,7 @@ int ide_queue_pc_tail(ide_drive_t *drive, struct gendisk *disk,
 
         rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
         ide_req(rq)->type = ATA_PRIV_MISC;
-        rq->special = (char *)pc;
+        ide_req(rq)->special = pc;
 
         if (buf && bufflen) {
                 error = blk_rq_map_kern(drive->queue, rq, buf, bufflen,
@@ -172,8 +172,8 @@ EXPORT_SYMBOL_GPL(ide_create_request_sense_cmd);
 void ide_prep_sense(ide_drive_t *drive, struct request *rq)
 {
         struct request_sense *sense = &drive->sense_data;
-        struct request *sense_rq = drive->sense_rq;
-        struct scsi_request *req = scsi_req(sense_rq);
+        struct request *sense_rq;
+        struct scsi_request *req;
         unsigned int cmd_len, sense_len;
         int err;
 
@@ -196,9 +196,16 @@ void ide_prep_sense(ide_drive_t *drive, struct request *rq)
         if (ata_sense_request(rq) || drive->sense_rq_armed)
                 return;
 
+        sense_rq = drive->sense_rq;
+        if (!sense_rq) {
+                sense_rq = blk_mq_alloc_request(drive->queue, REQ_OP_DRV_IN,
+                                        BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
+                drive->sense_rq = sense_rq;
+        }
+        req = scsi_req(sense_rq);
+
         memset(sense, 0, sizeof(*sense));
 
-        blk_rq_init(rq->q, sense_rq);
         scsi_req_init(req);
 
         err = blk_rq_map_kern(drive->queue, sense_rq, sense, sense_len,
@@ -207,6 +214,8 @@ void ide_prep_sense(ide_drive_t *drive, struct request *rq)
                 if (printk_ratelimit())
                         printk(KERN_WARNING PFX "%s: failed to map sense "
                                             "buffer\n", drive->name);
+                blk_mq_free_request(sense_rq);
+                drive->sense_rq = NULL;
                 return;
         }
 
@@ -226,6 +235,8 @@ EXPORT_SYMBOL_GPL(ide_prep_sense);
 
 int ide_queue_sense_rq(ide_drive_t *drive, void *special)
 {
+        struct request *sense_rq = drive->sense_rq;
+
         /* deferred failure from ide_prep_sense() */
         if (!drive->sense_rq_armed) {
                 printk(KERN_WARNING PFX "%s: error queuing a sense request\n",
@@ -233,12 +244,12 @@ int ide_queue_sense_rq(ide_drive_t *drive, void *special)
                 return -ENOMEM;
         }
 
-        drive->sense_rq->special = special;
+        ide_req(sense_rq)->special = special;
         drive->sense_rq_armed = false;
 
         drive->hwif->rq = NULL;
 
-        elv_add_request(drive->queue, drive->sense_rq, ELEVATOR_INSERT_FRONT);
+        ide_insert_request_head(drive, sense_rq);
         return 0;
 }
 EXPORT_SYMBOL_GPL(ide_queue_sense_rq);
@@ -270,10 +281,8 @@ void ide_retry_pc(ide_drive_t *drive)
          */
         drive->hwif->rq = NULL;
         ide_requeue_and_plug(drive, failed_rq);
-        if (ide_queue_sense_rq(drive, pc)) {
-                blk_start_request(failed_rq);
+        if (ide_queue_sense_rq(drive, pc))
                 ide_complete_rq(drive, BLK_STS_IOERR, blk_rq_bytes(failed_rq));
-        }
 }
 EXPORT_SYMBOL_GPL(ide_retry_pc);
 
diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c
index f9b59d41813f..1f03884a6808 100644
--- a/drivers/ide/ide-cd.c
+++ b/drivers/ide/ide-cd.c
@@ -211,12 +211,12 @@ static void cdrom_analyze_sense_data(ide_drive_t *drive,
 static void ide_cd_complete_failed_rq(ide_drive_t *drive, struct request *rq)
 {
         /*
-         * For ATA_PRIV_SENSE, "rq->special" points to the original
+         * For ATA_PRIV_SENSE, "ide_req(rq)->special" points to the original
          * failed request.  Also, the sense data should be read
          * directly from rq which might be different from the original
          * sense buffer if it got copied during mapping.
          */
-        struct request *failed = (struct request *)rq->special;
+        struct request *failed = ide_req(rq)->special;
         void *sense = bio_data(rq->bio);
 
         if (failed) {
@@ -258,11 +258,22 @@ static int ide_cd_breathe(ide_drive_t *drive, struct request *rq)
                 /*
                  * take a breather
                  */
-                blk_delay_queue(drive->queue, 1);
+                blk_mq_requeue_request(rq, false);
+                blk_mq_delay_kick_requeue_list(drive->queue, 1);
                 return 1;
         }
 }
 
+static void ide_cd_free_sense(ide_drive_t *drive)
+{
+        if (!drive->sense_rq)
+                return;
+
+        blk_mq_free_request(drive->sense_rq);
+        drive->sense_rq = NULL;
+        drive->sense_rq_armed = false;
+}
+
 /**
  * Returns:
  * 0: if the request should be continued.
@@ -516,6 +527,82 @@ static bool ide_cd_error_cmd(ide_drive_t *drive, struct ide_cmd *cmd)
         return false;
 }
 
+/* standard prep_rq that builds 10 byte cmds */
+static bool ide_cdrom_prep_fs(struct request_queue *q, struct request *rq)
+{
+        int hard_sect = queue_logical_block_size(q);
+        long block = (long)blk_rq_pos(rq) / (hard_sect >> 9);
+        unsigned long blocks = blk_rq_sectors(rq) / (hard_sect >> 9);
+        struct scsi_request *req = scsi_req(rq);
+
+        if (rq_data_dir(rq) == READ)
+                req->cmd[0] = GPCMD_READ_10;
+        else
+                req->cmd[0] = GPCMD_WRITE_10;
+
+        /*
+         * fill in lba
+         */
+        req->cmd[2] = (block >> 24) & 0xff;
+        req->cmd[3] = (block >> 16) & 0xff;
+        req->cmd[4] = (block >>  8) & 0xff;
+        req->cmd[5] = block & 0xff;
+
+        /*
+         * and transfer length
+         */
+        req->cmd[7] = (blocks >> 8) & 0xff;
+        req->cmd[8] = blocks & 0xff;
+        req->cmd_len = 10;
+        return true;
+}
+
+/*
+ * Most of the SCSI commands are supported directly by ATAPI devices.
+ * This transform handles the few exceptions.
+ */
+static bool ide_cdrom_prep_pc(struct request *rq)
+{
+        u8 *c = scsi_req(rq)->cmd;
+
+        /* transform 6-byte read/write commands to the 10-byte version */
+        if (c[0] == READ_6 || c[0] == WRITE_6) {
+                c[8] = c[4];
+                c[5] = c[3];
+                c[4] = c[2];
+                c[3] = c[1] & 0x1f;
+                c[2] = 0;
+                c[1] &= 0xe0;
+                c[0] += (READ_10 - READ_6);
+                scsi_req(rq)->cmd_len = 10;
+                return true;
+        }
+
+        /*
+         * it's silly to pretend we understand 6-byte sense commands, just
+         * reject with ILLEGAL_REQUEST and the caller should take the
+         * appropriate action
+         */
+        if (c[0] == MODE_SENSE || c[0] == MODE_SELECT) {
+                scsi_req(rq)->result = ILLEGAL_REQUEST;
+                return false;
+        }
+
+        return true;
+}
+
+static bool ide_cdrom_prep_rq(ide_drive_t *drive, struct request *rq)
+{
+        if (!blk_rq_is_passthrough(rq)) {
+                scsi_req_init(scsi_req(rq));
+
+                return ide_cdrom_prep_fs(drive->queue, rq);
+        } else if (blk_rq_is_scsi(rq))
+                return ide_cdrom_prep_pc(rq);
+
+        return true;
+}
+
 static ide_startstop_t cdrom_newpc_intr(ide_drive_t *drive)
 {
         ide_hwif_t *hwif = drive->hwif;
@@ -675,7 +762,7 @@ static ide_startstop_t cdrom_newpc_intr(ide_drive_t *drive)
 out_end:
         if (blk_rq_is_scsi(rq) && rc == 0) {
                 scsi_req(rq)->resid_len = 0;
-                blk_end_request_all(rq, BLK_STS_OK);
+                blk_mq_end_request(rq, BLK_STS_OK);
                 hwif->rq = NULL;
         } else {
                 if (sense && uptodate)
@@ -705,6 +792,8 @@ out_end:
                 if (sense && rc == 2)
                         ide_error(drive, "request sense failure", stat);
         }
+
+        ide_cd_free_sense(drive);
         return ide_stopped;
 }
 
@@ -729,7 +818,7 @@ static ide_startstop_t cdrom_start_rw(ide_drive_t *drive, struct request *rq)
                  * We may be retrying this request after an error.  Fix up any
                  * weirdness which might be present in the request packet.
                  */
-                q->prep_rq_fn(q, rq);
+                ide_cdrom_prep_rq(drive, rq);
         }
 
         /* fs requests *must* be hardware frame aligned */
@@ -1323,82 +1412,6 @@ static int ide_cdrom_probe_capabilities(ide_drive_t *drive)
         return nslots;
 }
 
-/* standard prep_rq_fn that builds 10 byte cmds */
-static int ide_cdrom_prep_fs(struct request_queue *q, struct request *rq)
-{
-        int hard_sect = queue_logical_block_size(q);
-        long block = (long)blk_rq_pos(rq) / (hard_sect >> 9);
-        unsigned long blocks = blk_rq_sectors(rq) / (hard_sect >> 9);
-        struct scsi_request *req = scsi_req(rq);
-
-        q->initialize_rq_fn(rq);
-
-        if (rq_data_dir(rq) == READ)
-                req->cmd[0] = GPCMD_READ_10;
-        else
-                req->cmd[0] = GPCMD_WRITE_10;
-
-        /*
-         * fill in lba
-         */
-        req->cmd[2] = (block >> 24) & 0xff;
-        req->cmd[3] = (block >> 16) & 0xff;
-        req->cmd[4] = (block >>  8) & 0xff;
-        req->cmd[5] = block & 0xff;
-
-        /*
-         * and transfer length
-         */
-        req->cmd[7] = (blocks >> 8) & 0xff;
-        req->cmd[8] = blocks & 0xff;
-        req->cmd_len = 10;
-        return BLKPREP_OK;
-}
-
-/*
- * Most of the SCSI commands are supported directly by ATAPI devices.
- * This transform handles the few exceptions.
- */
-static int ide_cdrom_prep_pc(struct request *rq)
-{
-        u8 *c = scsi_req(rq)->cmd;
-
-        /* transform 6-byte read/write commands to the 10-byte version */
-        if (c[0] == READ_6 || c[0] == WRITE_6) {
-                c[8] = c[4];
-                c[5] = c[3];
-                c[4] = c[2];
-                c[3] = c[1] & 0x1f;
-                c[2] = 0;
-                c[1] &= 0xe0;
-                c[0] += (READ_10 - READ_6);
-                scsi_req(rq)->cmd_len = 10;
-                return BLKPREP_OK;
-        }
-
-        /*
-         * it's silly to pretend we understand 6-byte sense commands, just
-         * reject with ILLEGAL_REQUEST and the caller should take the
-         * appropriate action
-         */
-        if (c[0] == MODE_SENSE || c[0] == MODE_SELECT) {
-                scsi_req(rq)->result = ILLEGAL_REQUEST;
-                return BLKPREP_KILL;
-        }
-
-        return BLKPREP_OK;
-}
-
-static int ide_cdrom_prep_fn(struct request_queue *q, struct request *rq)
-{
-        if (!blk_rq_is_passthrough(rq))
-                return ide_cdrom_prep_fs(q, rq);
-        else if (blk_rq_is_scsi(rq))
-                return ide_cdrom_prep_pc(rq);
-
-        return 0;
-}
-
 struct cd_list_entry {
         const char      *id_model;
         const char      *id_firmware;
@@ -1508,7 +1521,7 @@ static int ide_cdrom_setup(ide_drive_t *drive)
 
         ide_debug_log(IDE_DBG_PROBE, "enter");
 
-        blk_queue_prep_rq(q, ide_cdrom_prep_fn);
+        drive->prep_rq = ide_cdrom_prep_rq;
         blk_queue_dma_alignment(q, 31);
         blk_queue_update_dma_pad(q, 15);
 
@@ -1569,7 +1582,7 @@ static void ide_cd_release(struct device *dev)
         if (devinfo->handle == drive)
                 unregister_cdrom(devinfo);
         drive->driver_data = NULL;
-        blk_queue_prep_rq(drive->queue, NULL);
+        drive->prep_rq = NULL;
         g->private_data = NULL;
         put_disk(g);
         kfree(info);
diff --git a/drivers/ide/ide-devsets.c b/drivers/ide/ide-devsets.c
index f4f8afdf8bbe..f2f93ed40356 100644
--- a/drivers/ide/ide-devsets.c
+++ b/drivers/ide/ide-devsets.c
@@ -171,7 +171,7 @@ int ide_devset_execute(ide_drive_t *drive, const struct ide_devset *setting,
         scsi_req(rq)->cmd_len = 5;
         scsi_req(rq)->cmd[0] = REQ_DEVSET_EXEC;
         *(int *)&scsi_req(rq)->cmd[1] = arg;
-        rq->special = setting->set;
+        ide_req(rq)->special = setting->set;
 
         blk_execute_rq(q, NULL, rq, 0);
         ret = scsi_req(rq)->result;
@@ -182,7 +182,7 @@ int ide_devset_execute(ide_drive_t *drive, const struct ide_devset *setting,
 
 ide_startstop_t ide_do_devset(ide_drive_t *drive, struct request *rq)
 {
-        int err, (*setfunc)(ide_drive_t *, int) = rq->special;
+        int err, (*setfunc)(ide_drive_t *, int) = ide_req(rq)->special;
 
         err = setfunc(drive, *(int *)&scsi_req(rq)->cmd[1]);
         if (err)
diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c
index e3b4e659082d..197912af5c2f 100644
--- a/drivers/ide/ide-disk.c
+++ b/drivers/ide/ide-disk.c
@@ -427,16 +427,15 @@ static void ide_disk_unlock_native_capacity(ide_drive_t *drive)
                 drive->dev_flags |= IDE_DFLAG_NOHPA; /* disable HPA on resume */
 }
 
-static int idedisk_prep_fn(struct request_queue *q, struct request *rq)
+static bool idedisk_prep_rq(ide_drive_t *drive, struct request *rq)
 {
-        ide_drive_t *drive = q->queuedata;
         struct ide_cmd *cmd;
 
         if (req_op(rq) != REQ_OP_FLUSH)
-                return BLKPREP_OK;
+                return true;
 
-        if (rq->special) {
-                cmd = rq->special;
+        if (ide_req(rq)->special) {
+                cmd = ide_req(rq)->special;
                 memset(cmd, 0, sizeof(*cmd));
         } else {
                 cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC);
@@ -456,10 +455,10 @@ static int idedisk_prep_fn(struct request_queue *q, struct request *rq)
         rq->cmd_flags &= ~REQ_OP_MASK;
         rq->cmd_flags |= REQ_OP_DRV_OUT;
         ide_req(rq)->type = ATA_PRIV_TASKFILE;
-        rq->special = cmd;
+        ide_req(rq)->special = cmd;
         cmd->rq = rq;
 
-        return BLKPREP_OK;
+        return true;
 }
 
 ide_devset_get(multcount, mult_count);
@@ -548,7 +547,7 @@ static void update_flush(ide_drive_t *drive)
 
                 if (barrier) {
                         wc = true;
-                        blk_queue_prep_rq(drive->queue, idedisk_prep_fn);
+                        drive->prep_rq = idedisk_prep_rq;
                 }
         }
 
diff --git a/drivers/ide/ide-eh.c b/drivers/ide/ide-eh.c
index 47d5f3379748..e1323e058454 100644
--- a/drivers/ide/ide-eh.c
+++ b/drivers/ide/ide-eh.c
@@ -125,7 +125,7 @@ ide_startstop_t ide_error(ide_drive_t *drive, const char *msg, u8 stat)
         /* retry only "normal" I/O: */
         if (blk_rq_is_passthrough(rq)) {
                 if (ata_taskfile_request(rq)) {
-                        struct ide_cmd *cmd = rq->special;
+                        struct ide_cmd *cmd = ide_req(rq)->special;
 
                         if (cmd)
                                 ide_complete_cmd(drive, cmd, stat, err);
diff --git a/drivers/ide/ide-floppy.c b/drivers/ide/ide-floppy.c
index a8df300f949c..780d33ccc5d8 100644
--- a/drivers/ide/ide-floppy.c
+++ b/drivers/ide/ide-floppy.c
@@ -276,7 +276,7 @@ static ide_startstop_t ide_floppy_do_request(ide_drive_t *drive,
                 switch (ide_req(rq)->type) {
                 case ATA_PRIV_MISC:
                 case ATA_PRIV_SENSE:
-                        pc = (struct ide_atapi_pc *)rq->special;
+                        pc = (struct ide_atapi_pc *)ide_req(rq)->special;
                         break;
                 default:
                         BUG();
diff --git a/drivers/ide/ide-io.c b/drivers/ide/ide-io.c
index 0d93e0cfbeaf..8445b484ae69 100644
--- a/drivers/ide/ide-io.c
+++ b/drivers/ide/ide-io.c
@@ -67,7 +67,15 @@ int ide_end_rq(ide_drive_t *drive, struct request *rq, blk_status_t error,
                 ide_dma_on(drive);
         }
 
-        return blk_end_request(rq, error, nr_bytes);
+        if (!blk_update_request(rq, error, nr_bytes)) {
+                if (rq == drive->sense_rq)
+                        drive->sense_rq = NULL;
+
+                __blk_mq_end_request(rq, error);
+                return 0;
+        }
+
+        return 1;
 }
 EXPORT_SYMBOL_GPL(ide_end_rq);
 
@@ -103,7 +111,7 @@ void ide_complete_cmd(ide_drive_t *drive, struct ide_cmd *cmd, u8 stat, u8 err)
         }
 
         if (rq && ata_taskfile_request(rq)) {
-                struct ide_cmd *orig_cmd = rq->special;
+                struct ide_cmd *orig_cmd = ide_req(rq)->special;
 
                 if (cmd->tf_flags & IDE_TFLAG_DYN)
                         kfree(orig_cmd);
@@ -253,7 +261,7 @@ EXPORT_SYMBOL_GPL(ide_init_sg_cmd);
 static ide_startstop_t execute_drive_cmd (ide_drive_t *drive,
                 struct request *rq)
 {
-        struct ide_cmd *cmd = rq->special;
+        struct ide_cmd *cmd = ide_req(rq)->special;
 
         if (cmd) {
                 if (cmd->protocol == ATA_PROT_PIO) {
@@ -307,8 +315,6 @@ static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
 {
         ide_startstop_t startstop;
 
-        BUG_ON(!(rq->rq_flags & RQF_STARTED));
-
 #ifdef DEBUG
         printk("%s: start_request: current=0x%08lx\n",
                 drive->hwif->name, (unsigned long) rq);
@@ -320,6 +326,9 @@ static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
                 goto kill_rq;
         }
 
+        if (drive->prep_rq && !drive->prep_rq(drive, rq))
+                return ide_stopped;
+
         if (ata_pm_request(rq))
                 ide_check_pm_state(drive, rq);
 
@@ -343,7 +352,7 @@ static ide_startstop_t start_request (ide_drive_t *drive, struct request *rq)
                 if (ata_taskfile_request(rq))
                         return execute_drive_cmd(drive, rq);
                 else if (ata_pm_request(rq)) {
-                        struct ide_pm_state *pm = rq->special;
+                        struct ide_pm_state *pm = ide_req(rq)->special;
 #ifdef DEBUG_PM
                         printk("%s: start_power_step(step: %d)\n",
                                 drive->name, pm->pm_step);
@@ -430,44 +439,42 @@ static inline void ide_unlock_host(struct ide_host *host)
         }
 }
 
-static void __ide_requeue_and_plug(struct request_queue *q, struct request *rq)
-{
-        if (rq)
-                blk_requeue_request(q, rq);
-        if (rq || blk_peek_request(q)) {
-                /* Use 3ms as that was the old plug delay */
-                blk_delay_queue(q, 3);
-        }
-}
-
 void ide_requeue_and_plug(ide_drive_t *drive, struct request *rq)
 {
         struct request_queue *q = drive->queue;
-        unsigned long flags;
 
-        spin_lock_irqsave(q->queue_lock, flags);
-        __ide_requeue_and_plug(q, rq);
-        spin_unlock_irqrestore(q->queue_lock, flags);
+        /* Use 3ms as that was the old plug delay */
+        if (rq) {
+                blk_mq_requeue_request(rq, false);
+                blk_mq_delay_kick_requeue_list(q, 3);
+        } else
+                blk_mq_delay_run_hw_queue(q->queue_hw_ctx[0], 3);
 }
 
 /*
  * Issue a new request to a device.
  */
-void do_ide_request(struct request_queue *q)
+blk_status_t ide_queue_rq(struct blk_mq_hw_ctx *hctx,
+                          const struct blk_mq_queue_data *bd)
 {
-        ide_drive_t     *drive = q->queuedata;
+        ide_drive_t     *drive = hctx->queue->queuedata;
         ide_hwif_t      *hwif = drive->hwif;
         struct ide_host *host = hwif->host;
-        struct request  *rq = NULL;
+        struct request  *rq = bd->rq;
         ide_startstop_t startstop;
 
-        spin_unlock_irq(q->queue_lock);
+        if (!blk_rq_is_passthrough(rq) && !(rq->rq_flags & RQF_DONTPREP)) {
+                rq->rq_flags |= RQF_DONTPREP;
+                ide_req(rq)->special = NULL;
+        }
 
         /* HLD do_request() callback might sleep, make sure it's okay */
         might_sleep();
 
         if (ide_lock_host(host, hwif))
-                goto plug_device_2;
+                return BLK_STS_DEV_RESOURCE;
+
+        blk_mq_start_request(rq);
 
         spin_lock_irq(&hwif->lock);
 
@@ -503,21 +510,16 @@ repeat:
                 hwif->cur_dev = drive;
                 drive->dev_flags &= ~(IDE_DFLAG_SLEEPING | IDE_DFLAG_PARKED);
 
-                spin_unlock_irq(&hwif->lock);
-                spin_lock_irq(q->queue_lock);
                 /*
                  * we know that the queue isn't empty, but this can happen
-                 * if the q->prep_rq_fn() decides to kill a request
+                 * if ->prep_rq() decides to kill a request
                  */
-                if (!rq)
-                        rq = blk_fetch_request(drive->queue);
-
-                spin_unlock_irq(q->queue_lock);
-                spin_lock_irq(&hwif->lock);
-
                 if (!rq) {
-                        ide_unlock_port(hwif);
-                        goto out;
+                        rq = bd->rq;
+                        if (!rq) {
+                                ide_unlock_port(hwif);
+                                goto out;
+                        }
                 }
 
                 /*
@@ -551,23 +553,24 @@ repeat:
                 if (startstop == ide_stopped) {
                         rq = hwif->rq;
                         hwif->rq = NULL;
-                        goto repeat;
+                        if (rq)
+                                goto repeat;
+                        ide_unlock_port(hwif);
+                        goto out;
                 }
-        } else
-                goto plug_device;
+        } else {
+plug_device:
+                spin_unlock_irq(&hwif->lock);
+                ide_unlock_host(host);
+                ide_requeue_and_plug(drive, rq);
+                return BLK_STS_OK;
+        }
+
 out:
         spin_unlock_irq(&hwif->lock);
         if (rq == NULL)
                 ide_unlock_host(host);
-        spin_lock_irq(q->queue_lock);
-        return;
-
-plug_device:
-        spin_unlock_irq(&hwif->lock);
-        ide_unlock_host(host);
-plug_device_2:
-        spin_lock_irq(q->queue_lock);
-        __ide_requeue_and_plug(q, rq);
+        return BLK_STS_OK;
 }
 
 static int drive_is_ready(ide_drive_t *drive)
@@ -887,3 +890,16 @@ void ide_pad_transfer(ide_drive_t *drive, int write, int len)
         }
 }
 EXPORT_SYMBOL_GPL(ide_pad_transfer);
+
+void ide_insert_request_head(ide_drive_t *drive, struct request *rq)
+{
+        ide_hwif_t *hwif = drive->hwif;
+        unsigned long flags;
+
+        spin_lock_irqsave(&hwif->lock, flags);
+        list_add_tail(&rq->queuelist, &drive->rq_list);
+        spin_unlock_irqrestore(&hwif->lock, flags);
+
+        kblockd_schedule_work(&drive->rq_work);
+}
+EXPORT_SYMBOL_GPL(ide_insert_request_head);
diff --git a/drivers/ide/ide-park.c b/drivers/ide/ide-park.c
index 622f0edb3945..102aa3bc3e7f 100644
--- a/drivers/ide/ide-park.c
+++ b/drivers/ide/ide-park.c
@@ -27,7 +27,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout)
                 spin_unlock_irq(&hwif->lock);
 
                 if (start_queue)
-                        blk_run_queue(q);
+                        blk_mq_run_hw_queues(q, true);
                 return;
         }
         spin_unlock_irq(&hwif->lock);
@@ -36,7 +36,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout)
         scsi_req(rq)->cmd[0] = REQ_PARK_HEADS;
         scsi_req(rq)->cmd_len = 1;
         ide_req(rq)->type = ATA_PRIV_MISC;
-        rq->special = &timeout;
+        ide_req(rq)->special = &timeout;
         blk_execute_rq(q, NULL, rq, 1);
         rc = scsi_req(rq)->result ? -EIO : 0;
         blk_put_request(rq);
@@ -54,7 +54,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout)
         scsi_req(rq)->cmd[0] = REQ_UNPARK_HEADS;
         scsi_req(rq)->cmd_len = 1;
         ide_req(rq)->type = ATA_PRIV_MISC;
-        elv_add_request(q, rq, ELEVATOR_INSERT_FRONT);
+        ide_insert_request_head(drive, rq);
 
 out:
         return;
@@ -67,7 +67,7 @@ ide_startstop_t ide_do_park_unpark(ide_drive_t *drive, struct request *rq)
 
         memset(&cmd, 0, sizeof(cmd));
         if (scsi_req(rq)->cmd[0] == REQ_PARK_HEADS) {
-                drive->sleep = *(unsigned long *)rq->special;
+                drive->sleep = *(unsigned long *)ide_req(rq)->special;
                 drive->dev_flags |= IDE_DFLAG_SLEEPING;
                 tf->command = ATA_CMD_IDLEIMMEDIATE;
                 tf->feature = 0x44;
diff --git a/drivers/ide/ide-pm.c b/drivers/ide/ide-pm.c
index 59217aa1d1fb..192e6c65d34e 100644
--- a/drivers/ide/ide-pm.c
+++ b/drivers/ide/ide-pm.c
@@ -21,7 +21,7 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg)
         memset(&rqpm, 0, sizeof(rqpm));
         rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, 0);
         ide_req(rq)->type = ATA_PRIV_PM_SUSPEND;
-        rq->special = &rqpm;
+        ide_req(rq)->special = &rqpm;
         rqpm.pm_step = IDE_PM_START_SUSPEND;
         if (mesg.event == PM_EVENT_PRETHAW)
                 mesg.event = PM_EVENT_FREEZE;
@@ -40,32 +40,17 @@ int generic_ide_suspend(struct device *dev, pm_message_t mesg)
         return ret;
 }
 
-static void ide_end_sync_rq(struct request *rq, blk_status_t error)
-{
-        complete(rq->end_io_data);
-}
-
 static int ide_pm_execute_rq(struct request *rq)
 {
         struct request_queue *q = rq->q;
-        DECLARE_COMPLETION_ONSTACK(wait);
 
-        rq->end_io_data = &wait;
-        rq->end_io = ide_end_sync_rq;
-
-        spin_lock_irq(q->queue_lock);
         if (unlikely(blk_queue_dying(q))) {
                 rq->rq_flags |= RQF_QUIET;
                 scsi_req(rq)->result = -ENXIO;
-                __blk_end_request_all(rq, BLK_STS_OK);
-                spin_unlock_irq(q->queue_lock);
+                blk_mq_end_request(rq, BLK_STS_OK);
                 return -ENXIO;
         }
-        __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT);
-        __blk_run_queue_uncond(q);
-        spin_unlock_irq(q->queue_lock);
-
-        wait_for_completion_io(&wait);
+        blk_execute_rq(q, NULL, rq, true);
 
         return scsi_req(rq)->result ? -EIO : 0;
 }
@@ -79,6 +64,8 @@ int generic_ide_resume(struct device *dev)
         struct ide_pm_state rqpm;
         int err;
 
+        blk_mq_start_stopped_hw_queues(drive->queue, true);
+
         if (ide_port_acpi(hwif)) {
                 /* call ACPI _PS0 / _STM only once */
                 if ((drive->dn & 1) == 0 || pair == NULL) {
@@ -92,7 +79,7 @@ int generic_ide_resume(struct device *dev)
         memset(&rqpm, 0, sizeof(rqpm));
         rq = blk_get_request(drive->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_PREEMPT);
         ide_req(rq)->type = ATA_PRIV_PM_RESUME;
-        rq->special = &rqpm;
+        ide_req(rq)->special = &rqpm;
         rqpm.pm_step = IDE_PM_START_RESUME;
         rqpm.pm_state = PM_EVENT_ON;
 
@@ -111,7 +98,7 @@ int generic_ide_resume(struct device *dev)
 
 void ide_complete_power_step(ide_drive_t *drive, struct request *rq)
 {
-        struct ide_pm_state *pm = rq->special;
+        struct ide_pm_state *pm = ide_req(rq)->special;
 
 #ifdef DEBUG_PM
         printk(KERN_INFO "%s: complete_power_step(step: %d)\n",
@@ -141,7 +128,7 @@ void ide_complete_power_step(ide_drive_t *drive, struct request *rq)
 
 ide_startstop_t ide_start_power_step(ide_drive_t *drive, struct request *rq)
 {
-        struct ide_pm_state *pm = rq->special;
+        struct ide_pm_state *pm = ide_req(rq)->special;
         struct ide_cmd cmd = { };
 
         switch (pm->pm_step) {
@@ -213,8 +200,7 @@ out_do_tf:
 void ide_complete_pm_rq(ide_drive_t *drive, struct request *rq)
 {
         struct request_queue *q = drive->queue;
-        struct ide_pm_state *pm = rq->special;
-        unsigned long flags;
+        struct ide_pm_state *pm = ide_req(rq)->special;
 
         ide_complete_power_step(drive, rq);
         if (pm->pm_step != IDE_PM_COMPLETED)
@@ -224,22 +210,19 @@ void ide_complete_pm_rq(ide_drive_t *drive, struct request *rq)
         printk("%s: completing PM request, %s\n", drive->name,
                (ide_req(rq)->type == ATA_PRIV_PM_SUSPEND) ? "suspend" : "resume");
 #endif
-        spin_lock_irqsave(q->queue_lock, flags);
         if (ide_req(rq)->type == ATA_PRIV_PM_SUSPEND)
-                blk_stop_queue(q);
+                blk_mq_stop_hw_queues(q);
         else
                 drive->dev_flags &= ~IDE_DFLAG_BLOCKED;
-        spin_unlock_irqrestore(q->queue_lock, flags);
 
         drive->hwif->rq = NULL;
 
-        if (blk_end_request(rq, BLK_STS_OK, 0))
-                BUG();
+        blk_mq_end_request(rq, BLK_STS_OK);
 }
 
 void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
 {
-        struct ide_pm_state *pm = rq->special;
+        struct ide_pm_state *pm = ide_req(rq)->special;
 
         if (blk_rq_is_private(rq) &&
             ide_req(rq)->type == ATA_PRIV_PM_SUSPEND &&
@@ -260,7 +243,6 @@ void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
                 ide_hwif_t *hwif = drive->hwif;
                 const struct ide_tp_ops *tp_ops = hwif->tp_ops;
                 struct request_queue *q = drive->queue;
-                unsigned long flags;
                 int rc;
 #ifdef DEBUG_PM
                 printk("%s: Wakeup request inited, waiting for !BSY...\n", drive->name);
@@ -274,8 +256,6 @@ void ide_check_pm_state(ide_drive_t *drive, struct request *rq)
                 if (rc)
                         printk(KERN_WARNING "%s: drive not ready on wakeup\n", drive->name);
 
-                spin_lock_irqsave(q->queue_lock, flags);
-                blk_start_queue(q);
-                spin_unlock_irqrestore(q->queue_lock, flags);
+                blk_mq_start_hw_queues(q);
         }
 }
diff --git a/drivers/ide/ide-probe.c b/drivers/ide/ide-probe.c
index 3b75a7b7a284..63627be0811a 100644
--- a/drivers/ide/ide-probe.c
+++ b/drivers/ide/ide-probe.c
@@ -746,10 +746,16 @@ static void ide_initialize_rq(struct request *rq)
 {
         struct ide_request *req = blk_mq_rq_to_pdu(rq);
 
+        req->special = NULL;
         scsi_req_init(&req->sreq);
         req->sreq.sense = req->sense;
 }
 
+static const struct blk_mq_ops ide_mq_ops = {
+        .queue_rq               = ide_queue_rq,
+        .initialize_rq_fn       = ide_initialize_rq,
+};
+
 /*
  * init request queue
  */
@@ -759,6 +765,7 @@ static int ide_init_queue(ide_drive_t *drive)
         ide_hwif_t *hwif = drive->hwif;
         int max_sectors = 256;
         int max_sg_entries = PRD_ENTRIES;
+        struct blk_mq_tag_set *set;
 
         /*
          *      Our default set up assumes the normal IDE case,
@@ -767,19 +774,26 @@ static int ide_init_queue(ide_drive_t *drive)
          *      limits and LBA48 we could raise it but as yet
          *      do not.
          */
-        q = blk_alloc_queue_node(GFP_KERNEL, hwif_to_node(hwif), NULL);
-        if (!q)
+
+        set = &drive->tag_set;
+        set->ops = &ide_mq_ops;
+        set->nr_hw_queues = 1;
+        set->queue_depth = 32;
+        set->reserved_tags = 1;
+        set->cmd_size = sizeof(struct ide_request);
+        set->numa_node = hwif_to_node(hwif);
+        set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
+        if (blk_mq_alloc_tag_set(set))
                 return 1;
 
-        q->request_fn = do_ide_request;
-        q->initialize_rq_fn = ide_initialize_rq;
-        q->cmd_size = sizeof(struct ide_request);
-        blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, q);
-        if (blk_init_allocated_queue(q) < 0) {
-                blk_cleanup_queue(q);
+        q = blk_mq_init_queue(set);
+        if (IS_ERR(q)) {
+                blk_mq_free_tag_set(set);
                 return 1;
         }
 
+        blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, q);
+
         q->queuedata = drive;
         blk_queue_segment_boundary(q, 0xffff);
 
@@ -965,8 +979,12 @@ static void drive_release_dev (struct device *dev)
 
         ide_proc_unregister_device(drive);
 
+        if (drive->sense_rq)
+                blk_mq_free_request(drive->sense_rq);
+
         blk_cleanup_queue(drive->queue);
         drive->queue = NULL;
+        blk_mq_free_tag_set(&drive->tag_set);
 
         drive->dev_flags &= ~IDE_DFLAG_PRESENT;
 
@@ -1133,6 +1151,28 @@ static void ide_port_cable_detect(ide_hwif_t *hwif)
         }
 }
 
+/*
+ * Deferred request list insertion handler
+ */
+static void drive_rq_insert_work(struct work_struct *work)
+{
+        ide_drive_t *drive = container_of(work, ide_drive_t, rq_work);
+        ide_hwif_t *hwif = drive->hwif;
+        struct request *rq;
+        LIST_HEAD(list);
+
+        spin_lock_irq(&hwif->lock);
+        if (!list_empty(&drive->rq_list))
+                list_splice_init(&drive->rq_list, &list);
+        spin_unlock_irq(&hwif->lock);
+
+        while (!list_empty(&list)) {
+                rq = list_first_entry(&list, struct request, queuelist);
+                list_del_init(&rq->queuelist);
+                blk_execute_rq_nowait(drive->queue, rq->rq_disk, rq, true, NULL);
+        }
+}
+
 static const u8 ide_hwif_to_major[] =
         { IDE0_MAJOR, IDE1_MAJOR, IDE2_MAJOR, IDE3_MAJOR, IDE4_MAJOR,
           IDE5_MAJOR, IDE6_MAJOR, IDE7_MAJOR, IDE8_MAJOR, IDE9_MAJOR };
@@ -1145,12 +1185,10 @@ static void ide_port_init_devices_data(ide_hwif_t *hwif)
         ide_port_for_each_dev(i, drive, hwif) {
                 u8 j = (hwif->index * MAX_DRIVES) + i;
                 u16 *saved_id = drive->id;
-                struct request *saved_sense_rq = drive->sense_rq;
 
                 memset(drive, 0, sizeof(*drive));
                 memset(saved_id, 0, SECTOR_SIZE);
                 drive->id = saved_id;
-                drive->sense_rq = saved_sense_rq;
 
                 drive->media                    = ide_disk;
                 drive->select                   = (i << 4) | ATA_DEVICE_OBS;
@@ -1166,6 +1204,9 @@ static void ide_port_init_devices_data(ide_hwif_t *hwif)
 
                 INIT_LIST_HEAD(&drive->list);
                 init_completion(&drive->gendev_rel_comp);
+
+                INIT_WORK(&drive->rq_work, drive_rq_insert_work);
+                INIT_LIST_HEAD(&drive->rq_list);
         }
 }
 
@@ -1255,7 +1296,6 @@ static void ide_port_free_devices(ide_hwif_t *hwif)
         int i;
 
         ide_port_for_each_dev(i, drive, hwif) {
-                kfree(drive->sense_rq);
                 kfree(drive->id);
                 kfree(drive);
         }
@@ -1283,17 +1323,10 @@ static int ide_port_alloc_devices(ide_hwif_t *hwif, int node)
                 if (drive->id == NULL)
                         goto out_free_drive;
 
-                drive->sense_rq = kmalloc(sizeof(struct request) +
-                                sizeof(struct ide_request), GFP_KERNEL);
-                if (!drive->sense_rq)
-                        goto out_free_id;
-
                 hwif->devices[i] = drive;
         }
         return 0;
 
-out_free_id:
-        kfree(drive->id);
 out_free_drive:
         kfree(drive);
 out_nomem:
diff --git a/drivers/ide/ide-tape.c b/drivers/ide/ide-tape.c
index 34c1165226a4..db1a65f4b490 100644
--- a/drivers/ide/ide-tape.c
+++ b/drivers/ide/ide-tape.c
@@ -639,7 +639,7 @@ static ide_startstop_t idetape_do_request(ide_drive_t *drive,
                 goto out;
         }
         if (req->cmd[13] & REQ_IDETAPE_PC1) {
-                pc = (struct ide_atapi_pc *)rq->special;
+                pc = (struct ide_atapi_pc *)ide_req(rq)->special;
                 req->cmd[13] &= ~(REQ_IDETAPE_PC1);
                 req->cmd[13] |= REQ_IDETAPE_PC2;
                 goto out;
diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c
index c21d5c50ae3a..17b2e379e872 100644
--- a/drivers/ide/ide-taskfile.c
+++ b/drivers/ide/ide-taskfile.c
@@ -440,7 +440,7 @@ int ide_raw_taskfile(ide_drive_t *drive, struct ide_cmd *cmd, u8 *buf,
                         goto put_req;
         }
 
-        rq->special = cmd;
+        ide_req(rq)->special = cmd;
         cmd->rq = rq;
 
         blk_execute_rq(drive->queue, NULL, rq, 0);