16 files changed, 847 insertions, 821 deletions

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index b8c143d68ee0..fa8f26309444 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -1655,11 +1655,12 @@ static void blkiocg_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
         struct io_context *ioc;
 
         cgroup_taskset_for_each(task, cgrp, tset) {
-                task_lock(task);
-                ioc = task->io_context;
-                if (ioc)
-                        ioc->cgroup_changed = 1;
-                task_unlock(task);
+                /* we don't lose anything even if ioc allocation fails */
+                ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
+                if (ioc) {
+                        ioc_cgroup_changed(ioc);
+                        put_io_context(ioc, NULL);
+                }
         }
 }
 
diff --git a/block/blk-core.c b/block/blk-core.c
index 15de223c7f93..e6c05a97ee2b 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -39,6 +39,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
 
+DEFINE_IDA(blk_queue_ida);
+
 /*
  * For the allocated request tables
  */
@@ -358,7 +360,8 @@ EXPORT_SYMBOL(blk_put_queue);
 void blk_drain_queue(struct request_queue *q, bool drain_all)
 {
         while (true) {
-                int nr_rqs;
+                bool drain = false;
+                int i;
 
                 spin_lock_irq(q->queue_lock);
 
@@ -375,14 +378,25 @@ void blk_drain_queue(struct request_queue *q, bool drain_all)
                 if (!list_empty(&q->queue_head))
                         __blk_run_queue(q);
 
-                if (drain_all)
-                        nr_rqs = q->rq.count[0] + q->rq.count[1];
-                else
-                        nr_rqs = q->rq.elvpriv;
+                drain |= q->rq.elvpriv;
+
+                /*
+                 * Unfortunately, requests are queued at and tracked from
+                 * multiple places and there's no single counter which can
+                 * be drained.  Check all the queues and counters.
+                 */
+                if (drain_all) {
+                        drain |= !list_empty(&q->queue_head);
+                        for (i = 0; i < 2; i++) {
+                                drain |= q->rq.count[i];
+                                drain |= q->in_flight[i];
+                                drain |= !list_empty(&q->flush_queue[i]);
+                        }
+                }
 
                 spin_unlock_irq(q->queue_lock);
 
-                if (!nr_rqs)
+                if (!drain)
                         break;
                 msleep(10);
         }
@@ -469,6 +483,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         if (!q)
                 return NULL;
 
+        q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);
+        if (q->id < 0)
+                goto fail_q;
+
         q->backing_dev_info.ra_pages =
                         (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
         q->backing_dev_info.state = 0;
@@ -477,20 +495,17 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         q->node = node_id;
 
         err = bdi_init(&q->backing_dev_info);
-        if (err) {
-                kmem_cache_free(blk_requestq_cachep, q);
-                return NULL;
-        }
+        if (err)
+                goto fail_id;
 
-        if (blk_throtl_init(q)) {
-                kmem_cache_free(blk_requestq_cachep, q);
-                return NULL;
-        }
+        if (blk_throtl_init(q))
+                goto fail_id;
 
         setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
                     laptop_mode_timer_fn, (unsigned long) q);
         setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
         INIT_LIST_HEAD(&q->timeout_list);
+        INIT_LIST_HEAD(&q->icq_list);
         INIT_LIST_HEAD(&q->flush_queue[0]);
         INIT_LIST_HEAD(&q->flush_queue[1]);
         INIT_LIST_HEAD(&q->flush_data_in_flight);
@@ -508,6 +523,12 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         q->queue_lock = &q->__queue_lock;
 
         return q;
+
+fail_id:
+        ida_simple_remove(&blk_queue_ida, q->id);
+fail_q:
+        kmem_cache_free(blk_requestq_cachep, q);
+        return NULL;
 }
 EXPORT_SYMBOL(blk_alloc_queue_node);
 
@@ -605,26 +626,31 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
 }
 EXPORT_SYMBOL(blk_init_allocated_queue);
 
-int blk_get_queue(struct request_queue *q)
+bool blk_get_queue(struct request_queue *q)
 {
-        if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
-                kobject_get(&q->kobj);
-                return 0;
+        if (likely(!blk_queue_dead(q))) {
+                __blk_get_queue(q);
+                return true;
         }
 
-        return 1;
+        return false;
 }
 EXPORT_SYMBOL(blk_get_queue);
 
 static inline void blk_free_request(struct request_queue *q, struct request *rq)
 {
-        if (rq->cmd_flags & REQ_ELVPRIV)
+        if (rq->cmd_flags & REQ_ELVPRIV) {
                 elv_put_request(q, rq);
+                if (rq->elv.icq)
+                        put_io_context(rq->elv.icq->ioc, q);
+        }
+
         mempool_free(rq, q->rq.rq_pool);
 }
 
 static struct request *
-blk_alloc_request(struct request_queue *q, unsigned int flags, gfp_t gfp_mask)
+blk_alloc_request(struct request_queue *q, struct io_cq *icq,
+                  unsigned int flags, gfp_t gfp_mask)
 {
         struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
 
@@ -63