Merge branch 'blk-mq/core' into for-3.13/core

Signed-off-by: Jens Axboe <axboe@kernel.dk>

Conflicts:
	block/blk-timeout.c

28 files changed, 3721 insertions, 190 deletions

diff --git a/block/Makefile b/block/Makefile
index 671a83d063a5..20645e88fb57 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -5,8 +5,9 @@
 obj-$(CONFIG_BLOCK) := elevator.o blk-core.o blk-tag.o blk-sysfs.o \
                         blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
                         blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
-                        blk-iopoll.o blk-lib.o ioctl.o genhd.o scsi_ioctl.o \
-                        partition-generic.o partitions/
+                        blk-iopoll.o blk-lib.o blk-mq.o blk-mq-tag.o \
+                        blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \
+                        genhd.o scsi_ioctl.o partition-generic.o partitions/
 
 obj-$(CONFIG_BLK_DEV_BSG)       += bsg.o
 obj-$(CONFIG_BLK_DEV_BSGLIB)    += bsg-lib.o
diff --git a/block/blk-core.c b/block/blk-core.c
index 25f13479f552..8bdd0121212a 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -16,6 +16,7 @@
 #include <linux/backing-dev.h>
 #include <linux/bio.h>
 #include <linux/blkdev.h>
+#include <linux/blk-mq.h>
 #include <linux/highmem.h>
 #include <linux/mm.h>
 #include <linux/kernel_stat.h>
@@ -48,7 +49,7 @@ DEFINE_IDA(blk_queue_ida);
 /*
  * For the allocated request tables
  */
-static struct kmem_cache *request_cachep;
+struct kmem_cache *request_cachep = NULL;
 
 /*
  * For queue allocation
@@ -60,42 +61,6 @@ struct kmem_cache *blk_requestq_cachep;
  */
 static struct workqueue_struct *kblockd_workqueue;
 
-static void drive_stat_acct(struct request *rq, int new_io)
-{
-        struct hd_struct *part;
-        int rw = rq_data_dir(rq);
-        int cpu;
-
-        if (!blk_do_io_stat(rq))
-                return;
-
-        cpu = part_stat_lock();
-
-        if (!new_io) {
-                part = rq->part;
-                part_stat_inc(cpu, part, merges[rw]);
-        } else {
-                part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
-                if (!hd_struct_try_get(part)) {
-                        /*
-                         * The partition is already being removed,
-                         * the request will be accounted on the disk only
-                         *
-                         * We take a reference on disk->part0 although that
-                         * partition will never be deleted, so we can treat
-                         * it as any other partition.
-                         */
-                        part = &rq->rq_disk->part0;
-                        hd_struct_get(part);
-                }
-                part_round_stats(cpu, part);
-                part_inc_in_flight(part, rw);
-                rq->part = part;
-        }
-
-        part_stat_unlock();
-}
-
 void blk_queue_congestion_threshold(struct request_queue *q)
 {
         int nr;
@@ -145,7 +110,6 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
         rq->cmd = rq->__cmd;
         rq->cmd_len = BLK_MAX_CDB;
         rq->tag = -1;
-        rq->ref_count = 1;
         rq->start_time = jiffies;
         set_start_time_ns(rq);
         rq->part = NULL;
@@ -174,9 +138,9 @@ void blk_dump_rq_flags(struct request *rq, char *msg)
 {
         int bit;
 
-        printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
+        printk(KERN_INFO "%s: dev %s: type=%x, flags=%llx\n", msg,
                 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
-                rq->cmd_flags);
+                (unsigned long long) rq->cmd_flags);
 
         printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
                (unsigned long long)blk_rq_pos(rq),
@@ -595,9 +559,12 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         if (!q)
                 return NULL;
 
+        if (percpu_counter_init(&q->mq_usage_counter, 0))
+                goto fail_q;
+
         q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
         if (q->id < 0)
-                goto fail_q;
+                goto fail_c;
 
         q->backing_dev_info.ra_pages =
                         (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
@@ -644,6 +611,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         q->bypass_depth = 1;
         __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
 
+        init_waitqueue_head(&q->mq_freeze_wq);
+
         if (blkcg_init_queue(q))
                 goto fail_bdi;
 
@@ -653,6 +622,8 @@ fail_bdi:
         bdi_destroy(&q->backing_dev_info);
 fail_id:
         ida_simple_remove(&blk_queue_ida, q->id);
+fail_c:
+        percpu_counter_destroy(&q->mq_usage_counter);
 fail_q:
         kmem_cache_free(blk_requestq_cachep, q);
         return NULL;
@@ -1119,7 +1090,8 @@ retry:
         goto retry;
 }
 
-struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
+static struct request *blk_old_get_request(struct request_queue *q, int rw,
+                gfp_t gfp_mask)
 {
         struct request *rq;