Merge branch 'for-3.3/core' of git://git.kernel.dk/linux-block

* 'for-3.3/core' of git://git.kernel.dk/linux-block: (37 commits)
  Revert "block: recursive merge requests"
  block: Stop using macro stubs for the bio data integrity calls
  blockdev: convert some macros to static inlines
  fs: remove unneeded plug in mpage_readpages()
  block: Add BLKROTATIONAL ioctl
  block: Introduce blk_set_stacking_limits function
  block: remove WARN_ON_ONCE() in exit_io_context()
  block: an exiting task should be allowed to create io_context
  block: ioc_cgroup_changed() needs to be exported
  block: recursive merge requests
  block, cfq: fix empty queue crash caused by request merge
  block, cfq: move icq creation and rq->elv.icq association to block core
  block, cfq: restructure io_cq creation path for io_context interface cleanup
  block, cfq: move io_cq exit/release to blk-ioc.c
  block, cfq: move icq cache management to block core
  block, cfq: move io_cq lookup to blk-ioc.c
  block, cfq: move cfqd->icq_list to request_queue and add request->elv.icq
  block, cfq: reorganize cfq_io_context into generic and cfq specific parts
  block: remove elevator_queue->ops
  block: reorder elevator switch sequence
  ...

Fix up conflicts in:
 - block/blk-cgroup.c
	Switch from can_attach_task to can_attach
 - block/cfq-iosched.c
	conflict with now removed cic index changes (we now use q->id instead)

16 files changed, 847 insertions, 821 deletions

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index b8c143d68ee..fa8f2630944 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -1655,11 +1655,12 @@ static void blkiocg_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
         struct io_context *ioc;
 
         cgroup_taskset_for_each(task, cgrp, tset) {
-                task_lock(task);
-                ioc = task->io_context;
-                if (ioc)
-                        ioc->cgroup_changed = 1;
-                task_unlock(task);
+                /* we don't lose anything even if ioc allocation fails */
+                ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
+                if (ioc) {
+                        ioc_cgroup_changed(ioc);
+                        put_io_context(ioc, NULL);
+                }
         }
 }
 
diff --git a/block/blk-core.c b/block/blk-core.c
index 15de223c7f9..e6c05a97ee2 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -39,6 +39,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
 EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
 
+DEFINE_IDA(blk_queue_ida);
+
 /*
  * For the allocated request tables
  */
@@ -358,7 +360,8 @@ EXPORT_SYMBOL(blk_put_queue);
 void blk_drain_queue(struct request_queue *q, bool drain_all)
 {
         while (true) {
-                int nr_rqs;
+                bool drain = false;
+                int i;
 
                 spin_lock_irq(q->queue_lock);
 
@@ -375,14 +378,25 @@ void blk_drain_queue(struct request_queue *q, bool drain_all)
                 if (!list_empty(&q->queue_head))
                         __blk_run_queue(q);
 
-                if (drain_all)
-                        nr_rqs = q->rq.count[0] + q->rq.count[1];
-                else
-                        nr_rqs = q->rq.elvpriv;
+                drain |= q->rq.elvpriv;
+
+                /*
+                 * Unfortunately, requests are queued at and tracked from
+                 * multiple places and there's no single counter which can
+                 * be drained.  Check all the queues and counters.
+                 */
+                if (drain_all) {
+                        drain |= !list_empty(&q->queue_head);
+                        for (i = 0; i < 2; i++) {
+                                drain |= q->rq.count[i];
+                                drain |= q->in_flight[i];
+                                drain |= !list_empty(&q->flush_queue[i]);
+                        }
+                }
 
                 spin_unlock_irq(q->queue_lock);
 
-                if (!nr_rqs)
+                if (!drain)
                         break;
                 msleep(10);
         }
@@ -469,6 +483,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         if (!q)
                 return NULL;
 
+        q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);
+        if (q->id < 0)
+                goto fail_q;
+
         q->backing_dev_info.ra_pages =
                         (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
         q->backing_dev_info.state = 0;
@@ -477,20 +495,17 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         q->node = node_id;
 
         err = bdi_init(&q->backing_dev_info);
-        if (err) {
-                kmem_cache_free(blk_requestq_cachep, q);
-                return NULL;
-        }
+        if (err)
+                goto fail_id;
 
-        if (blk_throtl_init(q)) {
-                kmem_cache_free(blk_requestq_cachep, q);
-                return NULL;
-        }
+        if (blk_throtl_init(q))
+                goto fail_id;
 
         setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
                     laptop_mode_timer_fn, (unsigned long) q);
         setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
         INIT_LIST_HEAD(&q->timeout_list);
+        INIT_LIST_HEAD(&q->icq_list);
         INIT_LIST_HEAD(&q->flush_queue[0]);
         INIT_LIST_HEAD(&q->flush_queue[1]);
         INIT_LIST_HEAD(&q->flush_data_in_flight);
@@ -508,6 +523,12 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
         q->queue_lock = &q->__queue_lock;
 
         return q;
+
+fail_id:
+        ida_simple_remove(&blk_queue_ida, q->id);
+fail_q:
+        kmem_cache_free(blk_requestq_cachep, q);
+        return NULL;
 }
 EXPORT_SYMBOL(blk_alloc_queue_node);
 
@@ -605,26 +626,31 @@ blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
 }
 EXPORT_SYMBOL(blk_init_allocated_queue);
 
-int blk_get_queue(struct request_queue *q)
+bool blk_get_queue(struct request_queue *q)
 {
-        if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
-                kobject_get(&q->kobj);
-                return 0;
+        if (likely(!blk_queue_dead(q))) {
+                __blk_get_queue(q);
+                return true;
         }
 
-        return 1;
+        return false;
 }
 EXPORT_SYMBOL(blk_get_queue);
 
 static inline void blk_free_request(struct request_queue *q, struct request *rq)
 {
-        if (rq->cmd_flags & REQ_ELVPRIV)
+        if (rq->cmd_flags & REQ_ELVPRIV) {
                 elv_put_request(q, rq);
+                if (rq->elv.icq)
+                        put_io_context(rq->elv.icq->ioc, q);
+        }
+
         mempool_free(rq, q->rq.rq_pool);
 }
 
 static struct request *
-blk_alloc_request(struct request_queue *q, unsigned int flags, gfp_t gfp_mask)
+blk_alloc_request(struct request_queue *q, struct io_cq *icq,
+                  unsigned int flags, gfp_t gfp_mask)
 {
         struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
 
@@ -635,10 +661,15 @@ blk_alloc_request(struct request_queue *q, unsigned int flags, gfp_t gfp_mask)
 
         rq->cmd_flags = flags | REQ_ALLOCED;
 
-        if ((flags & REQ_ELVPRIV) &&
-            unlikely(elv_set_request(q, rq, gfp_mask))) {
-                mempool_free(rq, q->rq.rq_pool);
-                return NULL;
+        if (flags & REQ_ELVPRIV) {
+                rq->elv.icq = icq;
+                if (unlikely(elv_set_request(q, rq, gfp_mask))) {
+                        mempool_free(rq, q->rq.rq_pool);
+                        return NULL;
+                }
+                /* @rq->elv.icq holds on to io_context until @rq is freed */
+                if (icq)
+                        get_io_context(icq->ioc);
         }
 
         return rq;
@@ -750,11 +781,17 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
 {
         struct request *rq = NULL;
         struct request_list *rl = &q->rq;
-        struct io_context *ioc = NULL;
+        struct elevator_type *et;
+        struct io_context *ioc;
+        struct io_cq *icq = NULL;
         const bool is_sync = rw_is_sync(rw_flags) != 0;
+        bool retried = false;
         int may_queue;
+retry:
+        et = q->elevator->type;
+        ioc = current->io_context;
 
-        if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
+        if (unlikely(blk_queue_dead(q)))
                 return NULL;
 
         may_queue = elv_may_queue(q, rw_flags);
@@ -763,7 +800,20 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
 
         if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
                 if (rl->count[is_sync]+1 >= q->nr_requests) {
-                        ioc = current_io_context(GFP_ATOMIC, q->node);
+                        /*
+                         * We want ioc to record batching state.  If it's
+                         * not already there, creating a new one requires
+                         * dropping queue_lock, which in turn requires
+                         * retesting conditions to avoid queue hang.
+                         */
+                        if (!ioc && !retried) {
+                                spin_unlock_irq(q->queue_lock);
+                                create_io_context(current, gfp_mask, q->node);
+                                spin_lock_irq(q->queue_lock);
+                                retried = true;
+                                goto retry;
+                        }
+
                         /*
                          * The queue will fill after this allocation, so set
                          * it as full, and mark this process as "batching".
@@ -799,17 +849,36 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
         rl->count[is_sync]++;
         rl->starved[is_sync] = 0;
 
+        /*
+         * Decide whether the new request will be managed by elevator.  If
+         * so, mark @rw_flags and increment elvpriv.  Non-zero elvpriv will
+         * prevent the current elevator from being destroyed until the new
+         * request is freed.  This guarantees icq's won't be destroyed and
+         * makes creating new ones safe.
+         *
+         * Also, lookup icq while holding queue_lock.  If it doesn't exist,
+         * it will be created after releasing queue_lock.
+         */
         if (blk_rq_should_init_elevator(bio) &&
             !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags)) {
                 rw_flags |= REQ_ELVPRIV;
                 rl->elvpriv++;
+                if (et->icq_cache && ioc)
+                        icq = ioc_lookup_icq(ioc, q);
         }
 
         if (blk_queue_io_stat(q))
                 rw_flags |= REQ_IO_STAT;
         spin_unlock_irq(q->queue_lock);
 
-        rq = blk_alloc_request(q, rw_flags, gfp_mask);
+        /* create icq if missing */
+        if (unlikely(et->icq_cache && !icq))
+                icq = ioc_create_icq(q, gfp_mask);
+
+        /* rqs are guaranteed to have icq on elv_set_request() if requested */
+        if (likely(!et->icq_cache || icq))
+                rq = blk_alloc_request(q, icq, rw_flags, gfp_mask);
+
         if (unlikely(!rq)) {
                 /*
                  * Allocation failed presumably due to memory. Undo anything
@@ -871,10 +940,9 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,
         rq = get_request(q, rw_flags, bio, GFP_NOIO);
         while (!rq) {
                 DEFINE_WAIT(wait);
-                struct io_context *ioc;
                 struct request_list *rl = &q->rq;
 
-                if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
+                if (unlikely(blk_queue_dead(q)))
                         return NULL;
 
                 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
@@ -891,8 +959,8 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,
                  * up to a big batch of them for a small period time.
                  * See ioc_batching, ioc_set_batching
                  */
-                ioc = current_io_context(GFP_NOIO, q->node);
-                ioc_set_batching(q, ioc);
+                create_io_context(current, GFP_NOIO, q->node);
+                ioc_set_batching(q, current->io_context);
 
                 spin_lock_irq(q->queue_lock);
                 finish_wait(&rl->wait[is_sync], &wait);
@@ -1009,54 +1077,6 @@ static void add_acct_request(struct request_queue *q, struct request *rq,
         __elv_add_request(q, rq, where);
 }
 
-/**
- * blk_insert_request - insert a special request into a request queue
- * @q:          request queue where request should be inserted
- * @rq:         request to be inserted
- * @at_head:    insert request at head or tail of queue
- * @data:       private data
- *
- * Description:
- *    Many block devices need to execute commands asynchronously, so they don't
- *    block the whole kernel from preemption during request execution.  This is
- *    accomplished normally by inserting aritficial requests tagged as
- *    REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them
- *    be scheduled for actual execution by the request queue.
- *
- *    We have the option of inserting the head or the tail of the queue.
- *    Typically we use the tail for new ioctls and so forth.  We use the head
- *    of the queue for things like a QUEUE_FULL message from a device, or a
- *    host that is unable to accept a particular command.
- */
-void blk_insert_request(struct request_queue *q, struct request *rq,
-                        int at_head, void *data)
-{
-        int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
-        unsigned long flags;
-
-        /*
-         * tell I/O scheduler that this isn't a regular read/write (ie it
-         * must not attempt merges on this) and that it acts as a soft
-         * barrier
-         */
-        rq->cmd_type = REQ_TYPE_SPECIAL;
-
-        rq->special = data;
-
-        spin_lock_irqsave(q->queue_lock, flags);
-
-        /*
-         * If command is tagged, release the tag
-         */
-        if (blk_rq_tagged(rq))
-                blk_queue_end_tag(q, rq);
-
-        add_acct_request(q, rq, where);
-        __blk_run_queue(q);
-        spin_unlock_irqrestore(q->queue_lock, flags);
-}
-EXPORT_SYMBOL(blk_insert_request);
-
 static void part_round_stats_single(int cpu, struct hd_struct *part,
                                     unsigned long now)
 {
@@ -1766,6 +1786,10 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
                 return -EIO;
 
         spin_lock_irqsave(q->queue_lock, flags);
+        if (unlikely(blk_queue_dead(q))) {
+                spin_unlock_irqrestore(q->queue_lock, flags);
+                return -ENODEV;
+        }
 
         /*
          * Submitting request must be dequeued before calling this function
@@ -2740,6 +2764,14 @@ static void queue_unplugged(struct request_queue *q, unsigned int depth,
         trace_block_unplug(q, depth, !from_schedule);
 
         /*
+         * Don't mess with dead queue.
+         */
+        if (unlikely(blk_queue_dead(q))) {
+                spin_unlock(q->queue_lock);
+                return;
+        }
+
+        /*
          * If we are punting this to kblockd, then we can safely drop
          * the queue_lock before waking kblockd (which needs to take
          * this lock).
@@ -2815,6 +2847,15 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
                         depth = 0;
                         spin_lock(q->queue_lock);
                 }
+
+                /*
+                 * Short-circuit if @q is dead
+                 */
+                if (unlikely(blk_queue_dead(q))) {
+                        __blk_end_request_all(rq, -ENODEV);
+                        continue;
+                }
+
                 /*
                  * rq is already accounted, so use raw insert
                  */
diff --git a/block/blk-exec.c b/block/blk-exec.c
index a1ebceb332f..fb2cbd55162 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -50,7 +50,11 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
 {
         int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
 
-        if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
+        WARN_ON(irqs_disabled());
+        spin_lock_irq(q->queue_lock);
+
+        if (unlikely(blk_queue_dead(q))) {
+                spin_unlock_irq(q->queue_lock);
                 rq->errors = -ENXIO;
                 if (rq->end_io)
                         rq->end_io(rq, rq->errors);
@@ -59,8 +63,6 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
 
         rq->rq_disk = bd_disk;
         rq->end_io = done;
-        WARN_ON(irqs_disabled());
-        spin_lock_irq(q->queue_lock);
         __elv_add_request(q, rq, where);
         __blk_run_queue(q);
         /* the queue is stopped so it won't be run */
diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index 6f9bbd97865..27a06e00eae 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -16,53 +16,214 @@
  */
 static struct kmem_cache *iocontext_cachep;
 
-static void cfq_dtor(struct io_context *ioc)
+/**
+ * get_io_context - increment reference count to io_context
+ * @ioc: io_context to get
+ *
+ * Increment reference count to @ioc.
+ */
+void get_io_context(struct io_context *ioc)
+{
+        BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
+        atomic_long_inc(&ioc->refcount);
+}
+EXPORT_SYMBOL(get_io_context);
+
+/*
+ * Releasing ioc may nest into another put_io_context() leading to nested
+ * fast path release.  As the ioc's can't be the same, this is okay but
+ * makes lockdep whine.  Keep track of nesting and use it as subclass.
+ */
+#ifdef CONFIG_LOCKDEP
+#define ioc_release_depth(q)            ((q) ? (q)->ioc_release_depth : 0)
+#define ioc_release_depth_inc(q)        (q)->ioc_release_depth++
+#define ioc_release_depth_dec(q)        (q)->ioc_release_depth--
+#else
+#define ioc_release_depth(q)            0
+#define ioc_release_depth_inc(q)        do { } while (0)
+#define ioc_release_depth_dec(q)        do { } while (0)
+#endif
+
+static void icq_free_icq_rcu(struct rcu_head *head)
+{
+        struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);
+
+        kmem_cache_free(icq->__rcu_icq_cache, icq);
+}
+
+/*
+ * Exit and free an icq.  Called with both ioc and q locked.
+ */
+static void ioc_exit_icq(struct io_cq *icq)
 {
-        if (!hlist_empty(&ioc->cic_list)) {
-                struct cfq_io_context *cic;
+        struct io_context *ioc = icq->ioc;
+        struct request_queue *q = icq->q;
+        struct elevator_type *et = q->elevator->type;
+
+        lockdep_assert_held(&ioc->lock);
+        lockdep_assert_held(q->queue_lock);
+
+        radix_tree_delete(&ioc->icq_tree, icq->q->id);
+        hlist_del_init(&icq->ioc_node);
+        list_del_init(&icq->q_node);
+
+        /*
+         * Both setting lookup hint to and clearing it from @icq are done
+         * under queue_lock.  If it's not pointing to @icq now, it never
+         * will.  Hint assignment itself can race safely.
+         */
+        if (rcu_dereference_raw(ioc->icq_hint) == icq)
+                rcu_assign_pointer(ioc->icq_hint, NULL);
 
-                cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context,
-                                                                cic_list);
-                cic->dtor(ioc);
+        if (et->ops.elevator_exit_icq_fn) {
+                ioc_release_depth_inc(q);
+                et->ops.elevator_exit_icq_fn(icq);
+                ioc_release_depth_dec(q);
         }
+
+        /*
+         * @icq->q might have gone away by the time RCU callback runs
+         * making it impossible to determine icq_cache.  Record it in @icq.
+         */
+        icq->__rcu_icq_cache = et->icq_cache;
+        call_rcu(&icq->__rcu_head, icq_free_icq_rcu);
 }
 
 /*
- * IO Context helper functions. put_io_context() returns 1 if there are no
- * more users of this io context, 0 otherwise.
+ * Slow path for ioc release in put_io_context().  Performs double-lock
+ * dancing to unlink all icq's and then frees ioc.
  */
-int put_io_context(struct io_context *ioc)
+static void ioc_release_fn(struct work_struct *work)
 {
-        if (ioc == NULL)
-                return 1;
+        struct io_context *ioc = container_of(work, struct io_context,
+                                              release_work);
+        struct request_queue *last_q = NULL;
 
-        BUG_ON(atomic_long_read(&ioc->refcount) == 0);
+        spin_lock_irq(&ioc->lock);
 
-        if (atomic_long_dec_and_test(&ioc->refcount)) {
-                rcu_read_lock();
-                cfq_dtor(ioc);
-                rcu_read_unlock();
+        while (!hlist_empty(&ioc->icq_list)) {
+                struct io_cq *icq = hlist_entry(ioc->icq_list.first,
+                                                struct io_cq, ioc_node);
+                struct request_queue *this_q = icq->q;
 
-                kmem_cache_free(iocontext_cachep, ioc);
-                return 1;
+                if (this_q != last_q) {
+                        /*
+                         * Need to switch to @this_q.  Once we release
+                         * @ioc->lock, it can go away along with @cic.
+                         * Hold on to it.
+                         */
+                        __blk_get_queue(this_q);
+
+                        /*
+                         * blk_put_queue() might sleep thanks to kobject
+                         * idiocy.  Always release both locks, put and
+                         * restart.
+                         */
+                        if (last_q) {
+                                spin_unlock(last_q->queue_lock);
+                                spin_unlock_irq(&ioc->lock);
+                                blk_put_queue(last_q);
+                        } else {
+                                spin_unlock_irq(&ioc->lock);
+                        }
+
+                        last_q = this_q;
+                        spin_lock_irq(this_q->queue_lock);
+                        spin_lock(&ioc->lock);
+                        continue;
+                }
+                ioc_exit_icq(icq);
         }
-        return 0;
+
+        if (last_q) {
+                spin_unlock(last_q->queue_lock);
+                spin_unlock_irq(&ioc->lock);
+                blk_put_queue(last_q);
+        } else {
+                spin_unlock_irq(&ioc->lock);
+        }
+
+        kmem_cache_free(iocontext_cachep, ioc);
 }
-EXPORT_SYMBOL(put_io_context);
 
-static void cfq_exit(struct io_context *ioc)
+/**
+ * put_io_context - put a reference of io_context
+ * @ioc: io_context to put
+ * @locked_q: request_queue the caller is holding queue_lock of (hint)
+ *
+ * Decrement reference count of @ioc and release it if the count reaches
+ * zero.  If the caller is holding queue_lock of a queue, it can indicate
+ * that with @locked_q.  This is an optimization hint and the caller is
+ * allowed to pass in %NULL even when it's holding a queue_lock.
+ */
+void put_io_context(struct io_context *ioc, struct request_queue *locked_q)
 {
-        rcu_read_lock();
+        struct request_queue *last_q = locked_q;
+        unsigned long flags;
+
+        if (ioc == NULL)
+                return;
+
+        BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
+        if (locked_q)
+                lockdep_assert_held(locked_q->queue_lock);
 
-        if (!hlist_empty(&ioc->cic_list)) {
-                struct cfq_io_context *cic;
+        if (!atomic_long_dec_and_test(&ioc->refcount))
+                return;
+
+        /*
+         * Destroy @ioc.  This is a bit messy because icq's are chained
+         * from both ioc and queue, and ioc->lock nests inside queue_lock.
+         * The inner ioc->lock should be held to walk our icq_list and then
+         * for each icq the outer matching queue_lock should be grabbed.
+         * ie. We need to do reverse-order double lock dancing.
+         *
+         * Another twist is that we are often called with one of the
+         * matching queue_locks held as indicated by @locked_q, which
+         * prevents performing double-lock dance for other queues.
+         *
+         * So, we do it in two stages.  The fast path uses the queue_lock
+         * the caller is holding and, if other queues need to be accessed,
+         * uses trylock to avoid introducing locking dependency.  This can
+         * handle most cases, especially if @ioc was performing IO on only
+         * single device.
+         *
+         * If trylock doesn't cut it, we defer to @ioc->release_work which
+         * can do all the double-locking dancing.
+         */
+        spin_lock_irqsave_nested(&ioc->lock, flags,
+                                 ioc_release_depth(locked_q));
 
-                cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context,
-                                                                cic_list);
-                cic->exit(ioc);
+        while (!hlist_empty(&ioc->icq_list)) {
+                struct io_cq *icq = hlist_entry(ioc->icq_list.first,
+                                                struct io_cq, ioc_node);
+                struct request_queue *this_q = icq->q;
+
+                if (this_q != last_q) {
+                        if (last_q && last_q != locked_q)
+                                spin_unlock(last_q->queue_lock);
+                        last_q = NULL;
+
+                        if (!spin_trylock(this_q->queue_lock))
+                                break;
+                        last_q = this_q;
+                        continue;
+                }
+                ioc_exit_icq(icq);
         }
-        rcu_read_unlock();
+
+        if (last_q && last_q != locked_q)
+                spin_unlock(last_q->queue_lock);
+
+        spin_unlock_irqrestore(&ioc->lock, flags);
+
+        /* if no icq is left, we're done; otherwise, kick release_work */
+        if (hlist_empty(&ioc->icq_list))
+                kmem_cache_free(iocontext_cachep, ioc);
+        else
+                schedule_work(&ioc->release_work);
 }
+EXPORT_SYMBOL(put_io_context);
 
 /* Called by the exiting task */
 void exit_io_context(struct task_struct *task)
@@ -74,86 +235,240 @@ void exit_io_context(struct task_struct *task)
         task->io_context = NULL;
         task_unlock(task);
 
-        if (atomic_dec_and_test(&ioc->nr_tasks))
-                cfq_exit(ioc);
+        atomic_dec(&ioc->nr_tasks);
+        put_io_context(ioc, NULL);
+}
+
+/**
+ * ioc_clear_queue - break any ioc association with the specified queue
+ * @q: request_queue being cleared
+ *
+ * Walk @q->icq_list and exit all io_cq's.  Must be called with @q locked.
+ */
+void ioc_clear_queue(struct request_queue *q)
+{
+        lockdep_assert_held(q->queue_lock);
+
+        while (!list_empty(&q->icq_list)) {
+                struct io_cq *icq = list_entry(q->icq_list.next,
+                                               struct io_cq, q_node);
+                struct io_context *ioc = icq->ioc;
 
-        put_io_context(ioc);
+                spin_lock(&ioc->lock);
+                ioc_exit_icq(icq);
+                spin_unlock(&ioc->lock);
+        }
 }
 
-struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
+void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,
+                                int node)
 {
         struct io_context *ioc;
 
-        ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
-        if (ioc) {
-                atomic_long_set(&ioc->refcount, 1);
-                atomic_set(&ioc->nr_tasks, 1);
-                spin_lock_init(&ioc->lock);
-                ioc->ioprio_changed = 0;
-                ioc->ioprio = 0;
-                ioc->last_waited = 0; /* doesn't matter... */
-                ioc->nr_batch_requests = 0; /* because this is 0 */
-                INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH);
-                INIT_HLIST_HEAD(&ioc->cic_list);
-                ioc->ioc_data = NULL;
-#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE)
-                ioc->cgroup_changed = 0;
-#endif
-        }
+        ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
+                                    node);
+        if (unlikely(!ioc))
+                return;
 
-        return ioc;
+        /* initialize */
+        atomic_long_set(&ioc->refcount, 1);
+        atomic_set(&ioc->nr_tasks, 1);
+        spin_lock_init(&ioc->lock);
+        INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH);
+        INIT_HLIST_HEAD(&ioc->icq_list);
+        INIT_WORK(&ioc->release_work, ioc_release_fn);
+
+        /*
+         * Try to install.  ioc shouldn't be installed if someone else
+         * already did or @task, which isn't %current, is exiting.  Note
+         * that we need to allow ioc creation on exiting %current as exit
+         * path may issue IOs from e.g. exit_files().  The exit path is
+         * responsible for not issuing IO after exit_io_context().
+         */
+        task_lock(task);
+        if (!task->io_context &&
+            (task == current || !(task->flags & PF_EXITING)))
+                task->io_context = ioc;
+        else
+                kmem_cache_free(iocontext_cachep, ioc);
+        task_unlock(task);
 }
 
-/*
- * If the current task has no IO context then create one and initialise it.
- * Otherwise, return its existing IO context.
+/**
+ * get_task_io_context - get io_context of a task
+ * @task: task of interest
+ * @gfp_flags: allocation flags, used if allocation is necessary
+ * @node: allocation node, used if allocation is necessary
  *
- * This returned IO context doesn't have a specifically elevated refcount,
- * but since the current task itself holds a reference, the context can be
- * used in general code, so long as it stays within `current` context.
+ * Return io_context of @task.  If it doesn't exist, it is created with
+ * @gfp_flags and @node.  The returned io_context has its reference count
+ * incremented.
+ *
+ * This function always goes through task_lock() and it's better to use
+ * %current->io_context + get_io_context() for %current.
  */
-struct io_context *current_io_context(gfp_t gfp_flags, int node)
+struct io_context *get_task_io_context(struct task_struct *task,
+                                       gfp_t gfp_flags, int node)
 {
-        struct task_struct *tsk = current;
-        struct io_context *ret;
-
-        ret = tsk->io_context;
-        if (likely(ret))
-                return ret;
-
-        ret = alloc_io_context(gfp_flags, node);
-        if (ret) {
-                /* make sure set_task_ioprio() sees the settings above */
-                smp_wmb();
-                tsk->io_context = ret;
-        }
+        struct io_context *ioc;
 
-        return ret;
+        might_sleep_if(gfp_flags & __GFP_WAIT);
+
+        do {
+                task_lock(task);
+                ioc = task->io_context;
+                if (likely(ioc)) {
+                        get_io_context(ioc);
+                        task_unlock(task);
+                        return ioc;
+                }
+                task_unlock(task);
+        } while (create_io_context(task, gfp_flags, node));
+
+        return NULL;
 }
+EXPORT_SYMBOL(get_task_io_context);
 
-/*
- * If the current task has no IO context then create one and initialise it.
- * If it does have a context, take a ref on it.
+/**
+ * ioc_lookup_icq - lookup io_cq from ioc
+ * @ioc: the associated io_context
+ * @q: the associated request_queue
  *
- * This is always called in the context of the task which submitted the I/O.
+ * Look up io_cq associated with @ioc - @q pair from @ioc.  Must be called
+ * with @q->queue_lock held.
  */
-struct io_context *get_io_context(gfp_t gfp_flags, int node)
+struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q)
 {
-        struct io_context *ioc = NULL;
+        struct io_cq *icq;
+
+        lockdep_assert_held(q->queue_lock);
 
         /*
-         * Check for unlikely race with exiting task. ioc ref count is
-         * zero when ioc is being detached.
+         * icq's are indexed from @ioc using radix tree and hint pointer,
+         * both of which are protected with RCU.  All removals are done
+         * holding both q and ioc locks, and we're holding q lock - if we
+         * find a icq which points to us, it's guaranteed to be valid.
          */
-        do {
-                ioc = current_io_context(gfp_flags, node);
-                if (unlikely(!ioc))
-                        break;
-        } while (!atomic_long_inc_not_zero(&ioc->refcount));
+        rcu_read_lock();
+        icq = rcu_dereference(ioc->icq_hint);
+        if (icq && icq->q == q)
+                goto out;
 
-        return ioc;
+        icq = radix_tree_lookup(&ioc->icq_tree, q->id);
+        if (icq && icq->q == q)
+                rcu_assign_pointer(ioc->icq_hint, icq); /* allowed to race */
+        else
+                icq = NULL;
+out:
+        rcu_read_unlock();
+        return icq;
 }
-EXPORT_SYMBOL(get_io_context);
+EXPORT_SYMBOL(ioc_lookup_icq);
+
+/**
+ * ioc_create_icq - create and link io_cq
+ * @q: request_queue of interest
+ * @gfp_mask: allocation mask
+ *
+ * Make sure io_cq linking %current->io_context and @q exists.  If either
+ * io_context and/or icq don't exist, they will be created using @gfp_mask.
+ *
+ * The caller is responsible for ensuring @ioc won't go away and @q is
+ * alive and will stay alive until this function returns.
+ */
+struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask)
+{
+        struct elevator_type *et = q->elevator->type;
+        struct io_context *ioc;
+        struct io_cq *icq;
+
+        /* allocate stuff */
+        ioc = create_io_context(current, gfp_mask, q->node);
+        if (!ioc)
+                return NULL;
+
+        icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
+                                    q->node);
+        if (!icq)
+                return NULL;
+
+        if (radix_tree_preload(gfp_mask) < 0) {
+                kmem_cache_free(et->icq_cache, icq);
+                return NULL;
+        }
+
+        icq->ioc = ioc;
+        icq->q = q;
+        INIT_LIST_HEAD(&icq->q_node);
+        INIT_HLIST_NODE(&icq->ioc_node);
+
+        /* lock both q and ioc and try to link @icq */
+        spin_lock_irq(q->queue_lock);
+        spin_lock(&ioc->lock);
+
+        if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
+                hlist_add_head(&icq->ioc_node, &ioc->icq_list);
+                list_add(&icq->q_node, &q->icq_list);
+                if (et->ops.elevator_init_icq_fn)
+                        et->ops.elevator_init_icq_fn(icq);
+        } else {
+                kmem_cache_free(et->icq_cache, icq);
+                icq = ioc_lookup_icq(ioc, q);
+                if (!icq)
+                        printk(KERN_ERR "cfq: icq link failed!\n");
+        }
+
+        spin_unlock(&ioc->lock);
+        spin_unlock_irq(q->queue_lock);
+        radix_tree_preload_end();
+        return icq;
+}
+
+void ioc_set_changed(struct io_context *ioc, int which)
+{
+        struct io_cq *icq;
+        struct hlist_node *n;
+
+        hlist_for_each_entry(icq, n, &ioc->icq_list, ioc_node)
+                set_bit(which, &icq->changed);
+}
+
+/**
+ * ioc_ioprio_changed - notify ioprio change
+ * @ioc: io_context of interest
+ * @ioprio: new ioprio
+ *
+ * @ioc's ioprio has changed to @ioprio.  Set %ICQ_IOPRIO_CHANGED for all
+ * icq's.  iosched is responsible for checking the bit and applying it on
+ * request issue path.
+ */
+void ioc_ioprio_changed(struct io_context *ioc, int ioprio)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&ioc->lock, flags);
+        ioc->ioprio = ioprio;
+        ioc_set_changed(ioc, ICQ_IOPRIO_CHANGED);
+        spin_unlock_irqrestore(&ioc->lock, flags);
+}
+
+/**
+ * ioc_cgroup_changed - notify cgroup change
+ * @ioc: io_context of interest
+ *
+ * @ioc's cgroup has changed.  Set %ICQ_CGROUP_CHANGED for all icq's.
+ * iosched is responsible for checking the bit and applying it on request
+ * issue path.
+ */
+void ioc_cgroup_changed(struct io_context *ioc)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&ioc->lock, flags);
+        ioc_set_changed(ioc, ICQ_CGROUP_CHANGED);
+        spin_unlock_irqrestore(&ioc->lock, flags);
+}
+EXPORT_SYMBOL(ioc_cgroup_changed);
 
 static int __init blk_ioc_init(void)
 {
diff --git a/block/blk-settings.c b/block/blk-settings.c
index fa1eb0449a0..d3234fc494a 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -104,9 +104,7 @@ EXPORT_SYMBOL_GPL(blk_queue_lld_busy);
  * @lim:  the queue_limits structure to reset
  *
  * Description:
- *   Returns a queue_limit struct to its default state.  Can be used by
- *   stacking drivers like DM that stage table swaps and reuse an
- *   existing device queue.
+ *   Returns a queue_limit struct to its default state.
  */
 void blk_set_default_limits(struct queue_limits *lim)
 {
@@ -114,13 +112,12 @@ void blk_set_default_limits(struct queue_limits *lim)
         lim->max_integrity_segments = 0;
         lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK;
         lim->max_segment_size = BLK_MAX_SEGMENT_SIZE;
-        lim->max_sectors = BLK_DEF_MAX_SECTORS;
-        lim->max_hw_sectors = INT_MAX;
+        lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS;
         lim->max_discard_sectors = 0;
         lim->discard_granularity = 0;
         lim->discard_alignment = 0;
         lim->discard_misaligned = 0;
-        lim->discard_zeroes_data = 1;
+        lim->discard_zeroes_data = 0;
         lim->logical_block_size = lim->physical_block_size = lim->io_min = 512;
         lim->bounce_pfn = (unsigned long)(BLK_BOUNCE_ANY >> PAGE_SHIFT);
         lim->alignment_offset = 0;
@@ -131,6 +128,27 @@ void blk_set_default_limits(struct queue_limits *lim)
 EXPORT_SYMBOL(blk_set_default_limits);
 
 /**
+ * blk_set_stacking_limits - set default limits for stacking devices
+ * @lim:  the queue_limits structure to reset
+ *
+ * Description:
+ *   Returns a queue_limit struct to its default state. Should be used
+ *   by stacking drivers like DM that have no internal limits.
+ */
+void blk_set_stacking_limits(struct queue_limits *lim)
+{
+        blk_set_default_limits(lim);
+
+        /* Inherit limits from component devices */
+        lim->discard_zeroes_data = 1;
+        lim->max_segments = USHRT_MAX;
+        lim->max_hw_sectors = UINT_MAX;
+
+        lim->max_sectors = BLK_DEF_MAX_SECTORS;
+}
+EXPORT_SYMBOL(blk_set_stacking_limits);
+
+/**
  * blk_queue_make_request - define an alternate make_request function for a device
  * @q:  the request queue for the device to be affected
  * @mfn: the alternate make_request function
@@ -165,8 +183,6 @@ void blk_queue_make_request(struct request_queue *q, make_request_fn *mfn)
         q->nr_batching = BLK_BATCH_REQ;
 
         blk_set_default_limits(&q->limits);
-        blk_queue_max_hw_sectors(q, BLK_SAFE_MAX_SECTORS);
-        q->limits.discard_zeroes_data = 0;
 
         /*
          * by default assume old behaviour and bounce for any highmem page
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index e7f9f657f10..cf150011d80 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -425,7 +425,7 @@ queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
         if (!entry->show)
                 return -EIO;
         mutex_lock(&q->sysfs_lock);
-        if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) {
+        if (blk_queue_dead(q)) {
                 mutex_unlock(&q->sysfs_lock);
                 return -ENOENT;
         }
@@ -447,7 +447,7 @@ queue_attr_store(struct kobject *kobj, struct attribute *attr,
 
         q = container_of(kobj, struct request_queue, kobj);
         mutex_lock(&q->sysfs_lock);
-        if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) {
+        if (blk_queue_dead(q)) {
                 mutex_unlock(&q->sysfs_lock);
                 return -ENOENT;
         }
@@ -479,8 +479,12 @@ static void blk_release_queue(struct kobject *kobj)
 
         blk_sync_queue(q);
 
-        if (q->elevator)
+        if (q->elevator) {
+                spin_lock_irq(q->queue_lock);
+                ioc_clear_queue(q);
+                spin_unlock_irq(q->queue_lock);
                 elevator_exit(q->elevator);
+        }
 
         blk_throtl_exit(q);
 
@@ -494,6 +498,8 @@ static void blk_release_queue(struct kobject *kobj)
         blk_trace_shutdown(q);
 
         bdi_destroy(&q->backing_dev_info);
+
+        ida_simple_remove(&blk_queue_ida, q->id);
         kmem_cache_free(blk_requestq_cachep, q);
 }
 
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 4553245d931..5eed6a76721 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -310,7 +310,7 @@ static struct throtl_grp * throtl_get_tg(struct throtl_data *td)
         struct request_queue *q = td->queue;
 
         /* no throttling for dead queue */
-        if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
+        if (unlikely(blk_queue_dead(q)))
                 return NULL;
 
         rcu_read_lock();
@@ -335,7 +335,7 @@ static struct throtl_grp * throtl_get_tg(struct throtl_data *td)
         spin_lock_irq(q->queue_lock);
 
         /* Make sure @q is still alive */
-        if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
+        if (unlikely(blk_queue_dead(q))) {
                 kfree(tg);
                 return NULL;
         }
diff --git a/block/blk.h b/block/blk.h
index 3f6551b3c92..7efd772336d 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -1,6 +1,8 @@
 #ifndef BLK_INTERNAL_H
 #define BLK_INTERNAL_H
 
+#include <linux/idr.h>
+
 /* Amount of time in which a process may batch requests */
 #define BLK_BATCH_TIME  (HZ/50UL)
 
@@ -9,6 +11,12 @@
 
 extern struct kmem_cache *blk_requestq_cachep;
 extern struct kobj_type blk_queue_ktype;
+extern struct ida blk_queue_ida;
+
+static inline void __blk_get_queue(struct request_queue *q)
+{
+        kobject_get(&q->kobj);
+}
 
 void init_request_from_bio(struct request *req, struct bio *bio);
 void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
@@ -85,8 +93,8 @@ static inline struct request *__elv_next_request(struct request_queue *q)
                         q->flush_queue_delayed = 1;
                         return NULL;
                 }
-                if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags) ||
-                    !q->elevator->ops->elevator_dispatch_fn(q, 0))
+                if (unlikely(blk_queue_dead(q)) ||
+                    !q->elevator->type->ops.elevator_dispatch_fn(q, 0))
                         return NULL;
         }
 }
@@ -95,16 +103,16 @@ static inline void elv_activate_rq(struct request_queue *q, struct request *rq)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_activate_req_fn)
-                e->ops->elevator_activate_req_fn(q, rq);
+        if (e->type->ops.elevator_activate_req_fn)
+                e->type->ops.elevator_activate_req_fn(q, rq);
 }
 
 static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_deactivate_req_fn)
-                e->ops->elevator_deactivate_req_fn(q, rq);
+        if (e->type->ops.elevator_deactivate_req_fn)
+                e->type->ops.elevator_deactivate_req_fn(q, rq);
 }
 
 #ifdef CONFIG_FAIL_IO_TIMEOUT
@@ -119,8 +127,6 @@ static inline int blk_should_fake_timeout(struct request_queue *q)
 }
 #endif
 
-struct io_context *current_io_context(gfp_t gfp_flags, int node);
-
 int ll_back_merge_fn(struct request_queue *q, struct request *req,
                      struct bio *bio);
 int ll_front_merge_fn(struct request_queue *q, struct request *req, 
@@ -189,6 +195,42 @@ static inline int blk_do_io_stat(struct request *rq)
                 (rq->cmd_flags & REQ_DISCARD));
 }
 
+/*
+ * Internal io_context interface
+ */
+void get_io_context(struct io_context *ioc);
+struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);
+struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask);
+void ioc_clear_queue(struct request_queue *q);
+
+void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_mask,
+                                int node);
+
+/**
+ * create_io_context - try to create task->io_context
+ * @task: target task
+ * @gfp_mask: allocation mask
+ * @node: allocation node
+ *
+ * If @task->io_context is %NULL, allocate a new io_context and install it.
+ * Returns the current @task->io_context which may be %NULL if allocation
+ * failed.
+ *
+ * Note that this function can't be called with IRQ disabled because
+ * task_lock which protects @task->io_context is IRQ-unsafe.
+ */
+static inline struct io_context *create_io_context(struct task_struct *task,
+                                                   gfp_t gfp_mask, int node)
+{
+        WARN_ON_ONCE(irqs_disabled());
+        if (unlikely(!task->io_context))
+                create_io_context_slowpath(task, gfp_mask, node);
+        return task->io_context;
+}
+
+/*
+ * Internal throttling interface
+ */
 #ifdef CONFIG_BLK_DEV_THROTTLING
 extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio);
 extern void blk_throtl_drain(struct request_queue *q);
diff --git a/block/bsg.c b/block/bsg.c
index 9651ec7b87c..4cf703fd98b 100644
--- a/block/bsg.c
+++ b/block/bsg.c
@@ -769,12 +769,10 @@ static struct bsg_device *bsg_add_device(struct inode *inode,
                                          struct file *file)
 {
         struct bsg_device *bd;
-        int ret;
 #ifdef BSG_DEBUG
         unsigned char buf[32];
 #endif
-        ret = blk_get_queue(rq);
-        if (ret)
+        if (!blk_get_queue(rq))
                 return ERR_PTR(-ENXIO);
 
         bd = bsg_alloc_device();
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 3548705b04e..163263ddd38 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -14,6 +14,7 @@
 #include <linux/rbtree.h>
 #include <linux/ioprio.h>
 #include <linux/blktrace_api.h>
+#include "blk.h"
 #include "cfq.h"
 
 /*
@@ -53,20 +54,11 @@ static const int cfq_hist_divisor = 4;
 #define CFQQ_SECT_THR_NONROT    (sector_t)(2 * 32)
 #define CFQQ_SEEKY(cfqq)        (hweight32(cfqq->seek_history) > 32/8)
 
-#define RQ_CIC(rq)              \
-        ((struct cfq_io_context *) (rq)->elevator_private[0])
-#define RQ_CFQQ(rq)             (struct cfq_queue *) ((rq)->elevator_private[1])
-#define RQ_CFQG(rq)             (struct cfq_group *) ((rq)->elevator_private[2])
+#define RQ_CIC(rq)              icq_to_cic((rq)->elv.icq)
+#define RQ_CFQQ(rq)             (struct cfq_queue *) ((rq)->elv.priv[0])
+#define RQ_CFQG(rq)             (struct cfq_group *) ((rq)->elv.priv[1])
 
 static struct kmem_cache *cfq_pool;
-static struct kmem_cache *cfq_ioc_pool;
-
-static DEFINE_PER_CPU(unsigned long, cfq_ioc_count);
-static struct completion *ioc_gone;
-static DEFINE_SPINLOCK(ioc_gone_lock);
-
-static DEFINE_SPINLOCK(cic_index_lock);
-static DEFINE_IDA(cic_index_ida);
 
 #define CFQ_PRIO_LISTS          IOPRIO_BE_NR
 #define cfq_class_idle(cfqq)    ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
@@ -75,6 +67,14 @@ static DEFINE_IDA(cic_index_ida);
 #define sample_valid(samples)   ((samples) > 80)
 #define rb_entry_cfqg(node)     rb_entry((node), struct cfq_group, rb_node)
 
+struct cfq_ttime {
+        unsigned long last_end_request;
+
+        unsigned long ttime_total;
+        unsigned long ttime_samples;
+        unsigned long ttime_mean;
+};
+
 /*
  * Most of our rbtree usage is for sorting with min extraction, so
  * if we cache the leftmost node we don't have to walk down the tree
@@ -216,6 +216,12 @@ struct cfq_group {
         struct cfq_ttime ttime;
 };
 
+struct cfq_io_cq {
+        struct io_cq            icq;            /* must be the first member */
+        struct cfq_queue        *cfqq[2];
+        struct cfq_ttime        ttime;
+};
+
 /*
  * Per block device queue structure
  */
@@ -267,7 +273,7 @@ struct cfq_data {
         struct work_struct unplug_work;
 
         struct cfq_queue *active_queue;
-        struct cfq_io_context *active_cic;
+        struct cfq_io_cq *active_cic;
 
         /*
          * async queue for each priority case
@@ -290,9 +296,6 @@ struct cfq_data {
         unsigned int cfq_group_idle;
         unsigned int cfq_latency;
 
-        unsigned int cic_index;
-        struct list_head cic_list;
-
         /*
          * Fallback dummy cfqq for extreme OOM conditions
          */
@@ -464,37 +467,35 @@ static inline int cfqg_busy_async_queues(struct cfq_data *cfqd,
 static void cfq_dispatch_insert(struct request_queue *, struct request *);
 static struct cfq_queue *cfq_get_queue(struct cfq_data *, bool,
                                        struct io_context *, gfp_t);
-static struct cfq_io_context *cfq_cic_lookup(struct cfq_data *,
-                                                struct io_context *);
 
-static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_context *cic,
-                                            bool is_sync)
+static inline struct cfq_io_cq *icq_to_cic(struct io_cq *icq)
 {
-        return cic->cfqq[is_sync];
+        /* cic->icq is the first member, %NULL will convert to %NULL */
+        return container_of(icq, struct cfq_io_cq, icq);
 }
 
-static inline void cic_set_cfqq(struct cfq_io_context *cic,
-                                struct cfq_queue *cfqq, bool is_sync)
+static inline struct cfq_io_cq *cfq_cic_lookup(struct cfq_data *cfqd,
+                                               struct io_context *ioc)
 {
-        cic->cfqq[is_sync] = cfqq;
+        if (ioc)
+                return icq_to_cic(ioc_lookup_icq(ioc, cfqd->queue));
+        return NULL;
 }
 
-#define CIC_DEAD_KEY    1ul
-#define CIC_DEAD_INDEX_SHIFT    1
-
-static inline void *cfqd_dead_key(struct cfq_data *cfqd)
+static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)
 {
-        return (void *)(cfqd->cic_index << CIC_DEAD_INDEX_SHIFT | CIC_DEAD_KEY);
+        return cic->cfqq[is_sync];
 }
 
-static inline struct cfq_data *cic_to_cfqd(struct cfq_io_context *cic)
+static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,
+                                bool is_sync)
 {
-        struct cfq_data *cfqd = cic->key;
-
-        if (unlikely((unsigned long) cfqd & CIC_DEAD_KEY))
-                return NULL;
+        cic->cfqq[is_sync] = cfqq;
+}
 
-        return cfqd;
+static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)
+{
+        return cic->icq.q->elevator->elevator_data;
 }
 
 /*
@@ -1561,7 +1562,7 @@ static struct request *
 cfq_find_rq_fmerge(struct cfq_data *cfqd, struct bio *bio)
 {
         struct task_struct *tsk = current;
-        struct cfq_io_context *cic;
+        struct cfq_io_cq *cic;
         struct cfq_queue *cfqq;
 
         cic = cfq_cic_lookup(cfqd, tsk->io_context);
@@ -1687,7 +1688,7 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq,
                            struct bio *bio)
 {
         struct cfq_data *cfqd = q->elevator->elevator_data;
-        struct cfq_io_context *cic;
+        struct cfq_io_cq *cic;
         struct cfq_queue *cfqq;
 
         /*
@@ -1697,12 +1698,19 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq,
                 return false;
 
         /*
-         * Lookup the cfqq that this bio will be queued with. Allow
-         * merge only if rq is queued there.
+         * Lookup the cfqq that this bio will be queued with and allow
+         * merge only if rq is queued there.  This function can be called
+         * from plug merge without queue_lock.  In such cases, ioc of @rq
+         * and %current are guaranteed to be equal.  Avoid lookup which
+         * requires queue_lock by using @rq's cic.
          */
-        cic = cfq_cic_lookup(cfqd, current->io_context);
-        if (!cic)
-                return false;
+        if (current->io_context == RQ_CIC(rq)->icq.ioc) {
+                cic = RQ_CIC(rq);
+        } else {
+                cic = cfq_cic_lookup(cfqd, current->io_context);
+                if (!cic)
+                        return false;
+        }
 
         cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));
         return cfqq == RQ_CFQQ(rq);
@@ -1786,7 +1794,7 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                 cfqd->active_queue = NULL;
 
         if (cfqd->active_cic) {
-                put_io_context(cfqd->active_cic->ioc);
+                put_io_context(cfqd->active_cic->icq.ioc, cfqd->queue);
                 cfqd->active_cic = NULL;
         }
 }
@@ -2006,7 +2014,7 @@ static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 static void cfq_arm_slice_timer(struct cfq_data *cfqd)
 {
         struct cfq_queue *cfqq = cfqd->active_queue;
-        struct cfq_io_context *cic;
+        struct cfq_io_cq *cic;
         unsigned long sl, group_idle = 0;
 
         /*
@@ -2041,7 +2049,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
          * task has exited, don't wait
          */
         cic = cfqd->active_cic;
-        if (!cic || !atomic_read(&cic->ioc->nr_tasks))
+        if (!cic || !atomic_read(&cic->icq.ioc->nr_tasks))
                 return;
 
         /*
@@ -2592,9 +2600,9 @@ static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq)
         cfq_dispatch_insert(cfqd->queue, rq);
 
         if (!cfqd->active_cic) {
-                struct cfq_io_context *cic = RQ_CIC(rq);
+                struct cfq_io_cq *cic = RQ_CIC(rq);
 
-                atomic_long_inc(&cic->ioc->refcount);
+                atomic_long_inc(&cic->icq.ioc->refcount);
                 cfqd->active_cic = cic;
         }
 
@@ -2677,84 +2685,6 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
         cfq_put_cfqg(cfqg);
 }
 
-/*
- * Call func for each cic attached to this ioc.
- */
-static void
-call_for_each_cic(struct io_context *ioc,
-                  void (*func)(struct io_context *, struct cfq_io_context *))
-{
-        struct cfq_io_context *cic;
-        struct hlist_node *n;
-
-        rcu_read_lock();
-
-        hlist_for_each_entry_rcu(cic, n, &ioc->cic_list, cic_list)
-                func(ioc, cic);
-
-        rcu_read_unlock();
-}
-
-static void cfq_cic_free_rcu(struct rcu_head *head)
-{
-        struct cfq_io_context *cic;
-
-        cic = container_of(head, struct cfq_io_context, rcu_head);
-
-        kmem_cache_free(cfq_ioc_pool, cic);
-        elv_ioc_count_dec(cfq_ioc_count);
-
-        if (ioc_gone) {
-                /*
-                 * CFQ scheduler is exiting, grab exit lock and check
-                 * the pending io context count. If it hits zero,
-                 * complete ioc_gone and set it back to NULL
-                 */
-                spin_lock(&ioc_gone_lock);
-                if (ioc_gone && !elv_ioc_count_read(cfq_ioc_count)) {
-                        complete(ioc_gone);
-                        ioc_gone = NULL;
-                }
-                spin_unlock(&ioc_gone_lock);
-        }
-}
-
-static void cfq_cic_free(struct cfq_io_context *cic)
-{
-        call_rcu(&cic->rcu_head, cfq_cic_free_rcu);
-}
-
-static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic)
-{
-        unsigned long flags;
-        unsigned long dead_key = (unsigned long) cic->key;
-
-        BUG_ON(!(dead_key & CIC_DEAD_KEY));
-
-        spin_lock_irqsave(&ioc->lock, flags);
-        radix_tree_delete(&ioc->radix_root, dead_key >> CIC_DEAD_INDEX_SHIFT);
-        hlist_del_rcu(&cic->cic_list);
-        spin_unlock_irqrestore(&ioc->lock, flags);
-
-        cfq_cic_free(cic);
-}
-
-/*
- * Must be called with rcu_read_lock() held or preemption otherwise disabled.
- * Only two callers of this - ->dtor() which is called with the rcu_read_lock(),
- * and ->trim() which is called with the task lock held
- */
-static void cfq_free_io_context(struct io_context *ioc)
-{
-        /*
-         * ioc->refcount is zero here, or we are called from elv_unregister(),
-         * so no more cic's are allowed to be linked into this ioc.  So it
-         * should be ok to iterate over the known list, we will see all cic's
-         * since no new ones are added.
-         */
-        call_for_each_cic(ioc, cic_free_func);
-}
-
 static void cfq_put_cooperator(struct cfq_queue *cfqq)
 {
         struct cfq_queue *__cfqq, *next;
@@ -2788,27 +2718,17 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
         cfq_put_queue(cfqq);
 }
 
-static void __cfq_exit_single_io_context(struct cfq_data *cfqd,
-                                         struct cfq_io_context *cic)
+static void cfq_init_icq(struct io_cq *icq)
 {
-        struct io_context *ioc = cic->ioc;
-
-        list_del_init(&cic->queue_list);
+        struct cfq_io_cq *cic = icq_to_cic(icq);
 
-        /*
-         * Make sure dead mark is seen for dead queues
-         */
-        smp_wmb();
-        cic->key = cfqd_dead_key(cfqd);
+        cic->ttime.last_end_request = jiffies;
+}
 
-        rcu_read_lock();
-        if (rcu_dereference(ioc->ioc_data) == cic) {
-                rcu_read_unlock();
-                spin_lock(&ioc->lock);
-                rcu_assign_pointer(ioc->ioc_data, NULL);
-                spin_unlock(&ioc->lock);
-        } else
-                rcu_read_unlock();
+static void cfq_exit_icq(struct io_cq *icq)
+{
+        struct cfq_io_cq *cic = icq_to_cic(icq);
+        struct cfq_data *cfqd = cic_to_cfqd(cic);
 
         if (cic->cfqq[BLK_RW_ASYNC]) {
                 cfq_exit_cfqq(cfqd, cic->cfqq[BLK_RW_ASYNC]);
@@ -2821,57 +2741,6 @@ static void __cfq_exit_single_io_context(struct cfq_data *cfqd,
         }
 }
 
-static void cfq_exit_single_io_context(struct io_context *ioc,
-                                       struct cfq_io_context *cic)
-{
-        struct cfq_data *cfqd = cic_to_cfqd(cic);
-
-        if (cfqd) {
-                struct request_queue *q = cfqd->queue;
-                unsigned long flags;
-
-                spin_lock_irqsave(q->queue_lock, flags);
-
-                /*
-                 * Ensure we get a fresh copy of the ->key to prevent
-                 * race between exiting task and queue
-                 */
-                smp_read_barrier_depends();
-                if (cic->key == cfqd)
-                        __cfq_exit_single_io_context(cfqd, cic);
-
-                spin_unlock_irqrestore(q->queue_lock, flags);
-        }
-}
-
-/*
- * The process that ioc belongs to has exited, we need to clean up
- * and put the internal structures we have that belongs to that process.
- */
-static void cfq_exit_io_context(struct io_context *ioc)
-{
-        call_for_each_cic(ioc, cfq_exit_single_io_context);
-}
-
-static struct cfq_io_context *
-cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
-{
-        struct cfq_io_context *cic;
-
-        cic = kmem_cache_alloc_node(cfq_ioc_pool, gfp_mask | __GFP_ZERO,
-                                                        cfqd->queue->node);
-        if (cic) {
-                cic->ttime.last_end_request = jiffies;
-                INIT_LIST_HEAD(&cic->queue_list);
-                INIT_HLIST_NODE(&cic->cic_list);
-                cic->dtor = cfq_free_io_context;
-                cic->exit = cfq_exit_io_context;
-                elv_ioc_count_inc(cfq_ioc_count);
-        }
-
-        return cic;
-}
-
 static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc)
 {
         struct task_struct *tsk = current;
@@ -2914,21 +2783,18 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc)
         cfq_clear_cfqq_prio_changed(cfqq);
 }
 
-static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic)
+static void changed_ioprio(struct cfq_io_cq *cic)
 {
         struct cfq_data *cfqd = cic_to_cfqd(cic);
         struct cfq_queue *cfqq;
-        unsigned long flags;
 
         if (unlikely(!cfqd))
                 return;
 
-        spin_lock_irqsave(cfqd->queue->queue_lock, flags);
-
         cfqq = cic->cfqq[BLK_RW_ASYNC];
         if (cfqq) {
                 struct cfq_queue *new_cfqq;
-                new_cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic->ioc,
+                new_cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic->icq.ioc,
                                                 GFP_ATOMIC);
                 if (new_cfqq) {
                         cic->cfqq[BLK_RW_ASYNC] = new_cfqq;
@@ -2939,14 +2805,6 @@ static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic)
         cfqq = cic->cfqq[BLK_RW_SYNC];
         if (cfqq)
                 cfq_mark_cfqq_prio_changed(cfqq);
-
-        spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
-}
-
-static void cfq_ioc_set_ioprio(struct io_context *ioc)
-{
-        call_for_each_cic(ioc, changed_ioprio);
-        ioc->ioprio_changed = 0;
 }
 
 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
@@ -2970,11 +2828,10 @@ static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 }
 
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
-static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic)
+static void changed_cgroup(struct cfq_io_cq *cic)
 {
         struct cfq_queue *sync_cfqq = cic_to_cfqq(cic, 1);
         struct cfq_data *cfqd = cic_to_cfqd(cic);
-        unsigned long flags;
         struct request_queue *q;
 
         if (unlikely(!cfqd))
@@ -2982,8 +2839,6 @@ static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic)
 
         q = cfqd->queue;
 
-        spin_lock_irqsave(q->queue_lock, flags);
-
         if (sync_cfqq) {
                 /*
                  * Drop reference to sync queue. A new sync queue will be
@@ -2993,14 +2848,6 @@ static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic)
                 cic_set_cfqq(cic, NULL, 1);
                 cfq_put_queue(sync_cfqq);
         }
-
-        spin_unlock_irqrestore(q->queue_lock, flags);
-}
-
-static void cfq_ioc_set_cgroup(struct io_context *ioc)
-{
-        call_for_each_cic(ioc, changed_cgroup);
-        ioc->cgroup_changed = 0;
 }
 #endif  /* CONFIG_CFQ_GROUP_IOSCHED */
 
@@ -3009,7 +2856,7 @@ cfq_find_alloc_queue(struct cfq_data *cfqd, bool is_sync,
                      struct io_context *ioc, gfp_t gfp_mask)
 {
         struct cfq_queue *cfqq, *new_cfqq = NULL;
-        struct cfq_io_context *cic;
+        struct cfq_io_cq *cic;
         struct cfq_group *cfqg;
 
 retry:
@@ -3100,160 +2947,6 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct io_context *ioc,
         return cfqq;
 }
 
-/*
- * We drop cfq io contexts lazily, so we may find a dead one.
- */
-static void
-cfq_drop_dead_cic(struct cfq_data *cfqd, struct io_context *ioc,
-                  struct cfq_io_context *cic)
-{
-        unsigned long flags;
-
-        WARN_ON(!list_empty(&cic->queue_list));
-        BUG_ON(cic->key != cfqd_dead_key(cfqd));
-
-        spin_lock_irqsave(&ioc->lock, flags);
-
-        BUG_ON(rcu_dereference_check(ioc->ioc_data,
-                lockdep_is_held(&ioc->lock)) == cic);
-
-        radix_tree_delete(&ioc->radix_root, cfqd->cic_index);
-        hlist_del_rcu(&cic->cic_list);
-        spin_unlock_irqrestore(&ioc->lock, flags);
-
-        cfq_cic_free(cic);
-}
-
-static struct cfq_io_context *
-cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc)
-{
-        struct cfq_io_context *cic;
-        unsigned long flags;
-
-        if (unlikely(!ioc))
-                return NULL;
-
-        rcu_read_lock();
-
-        /*
-         * we maintain a last-hit cache, to avoid browsing over the tree
-         */
-        cic = rcu_dereference(ioc->ioc_data);
-        if (cic && cic->key == cfqd) {
-                rcu_read_unlock();
-                return cic;
-        }
-
-        do {
-                cic = radix_tree_lookup(&ioc->radix_root, cfqd->cic_index);
-                rcu_read_unlock();
-                if (!cic)
-                        break;
-                if (unlikely(cic->key != cfqd)) {
-                        cfq_drop_dead_cic(cfqd, ioc, cic);
-                        rcu_read_lock();
-                        continue;
-                }
-
-                spin_lock_irqsave(&ioc->lock, flags);
-                rcu_assign_pointer(ioc->ioc_data, cic);
-                spin_unlock_irqrestore(&ioc->lock, flags);
-                break;
-        } while (1);
-
-        return cic;
-}
-
-/*
- * Add cic into ioc, using cfqd as the search key. This enables us to lookup
- * the process specific cfq io context when entered from the block layer.
- * Also adds the cic to a per-cfqd list, used when this queue is removed.
- */
-static int cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc,
-                        struct cfq_io_context *cic, gfp_t gfp_mask)
-{
-        unsigned long flags;
-        int ret;
-
-        ret = radix_tree_preload(gfp_mask);
-        if (!ret) {
-                cic->ioc = ioc;
-                cic->key = cfqd;
-
-                spin_lock_irqsave(&ioc->lock, flags);
-                ret = radix_tree_insert(&ioc->radix_root,
-                                                cfqd->cic_index, cic);
-                if (!ret)
-                        hlist_add_head_rcu(&cic->cic_list, &ioc->cic_list);
-                spin_unlock_irqrestore(&ioc->lock, flags);
-
-                radix_tree_preload_end();
-
-                if (!ret) {
-                        spin_lock_irqsave(cfqd->queue->queue_lock, flags);
-                        list_add(&cic->queue_list, &cfqd->cic_list);
-                        spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
-                }
-        }
-
-        if (ret && ret != -EEXIST)
-                printk(KERN_ERR "cfq: cic link failed!\n");
-
-        return ret;
-}
-
-/*
- * Setup general io context and cfq io context. There can be several cfq
- * io contexts per general io context, if this process is doing io to more
- * than one device managed by cfq.
- */
-static struct cfq_io_context *
-cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
-{
-        struct io_context *ioc = NULL;
-        struct cfq_io_context *cic;
-        int ret;
-
-        might_sleep_if(gfp_mask & __GFP_WAIT);
-
-        ioc = get_io_context(gfp_mask, cfqd->queue->node);
-        if (!ioc)
-                return NULL;
-
-retry:
-        cic = cfq_cic_lookup(cfqd, ioc);
-        if (cic)
-                goto out;
-
-        cic = cfq_alloc_io_context(cfqd, gfp_mask);
-        if (cic == NULL)
-                goto err;
-
-        ret = cfq_cic_link(cfqd, ioc, cic, gfp_mask);
-        if (ret == -EEXIST) {
-                /* someone has linked cic to ioc already */
-                cfq_cic_free(cic);
-                goto retry;
-        } else if (ret)
-                goto err_free;
-
-out:
-        smp_read_barrier_depends();
-        if (unlikely(ioc->ioprio_changed))
-                cfq_ioc_set_ioprio(ioc);
-
-#ifdef CONFIG_CFQ_GROUP_IOSCHED
-        if (unlikely(ioc->cgroup_changed))
-                cfq_ioc_set_cgroup(ioc);
-#endif
-        return cic;
-err_free:
-        cfq_cic_free(cic);
-err:
-        put_io_context(ioc);
-        return NULL;
-}
-
 static void
 __cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle)
 {
@@ -3267,7 +2960,7 @@ __cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle)
 
 static void
 cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
-        struct cfq_io_context *cic)
+                        struct cfq_io_cq *cic)
 {
         if (cfq_cfqq_sync(cfqq)) {
                 __cfq_update_io_thinktime(&cic->ttime, cfqd->cfq_slice_idle);
@@ -3305,7 +2998,7 @@ cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
  */
 static void
 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,
-                       struct cfq_io_context *cic)
+                       struct cfq_io_cq *cic)
 {
         int old_idle, enable_idle;
 
@@ -3322,8 +3015,9 @@ cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 
         if (cfqq->next_rq && (cfqq->next_rq->cmd_flags & REQ_NOIDLE))
                 enable_idle = 0;
-        else if (!atomic_read(&cic->ioc->nr_tasks) || !cfqd->cfq_slice_idle ||
-            (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq)))
+        else if (!atomic_read(&cic->icq.ioc->nr_tasks) ||
+                 !cfqd->cfq_slice_idle ||
+                 (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq)))
                 enable_idle = 0;
         else if (sample_valid(cic->ttime.ttime_samples)) {
                 if (cic->ttime.ttime_mean > cfqd->cfq_slice_idle)
@@ -3455,7 +3149,7 @@ static void
 cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                 struct request *rq)
 {
-        struct cfq_io_context *cic = RQ_CIC(rq);
+        struct cfq_io_cq *cic = RQ_CIC(rq);
 
         cfqd->rq_queued++;
         if (rq->cmd_flags & REQ_PRIO)
@@ -3508,7 +3202,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)
         struct cfq_queue *cfqq = RQ_CFQQ(rq);
 
         cfq_log_cfqq(cfqd, cfqq, "insert_request");
-        cfq_init_prio_data(cfqq, RQ_CIC(rq)->ioc);
+        cfq_init_prio_data(cfqq, RQ_CIC(rq)->icq.ioc);
 
         rq_set_fifo_time(rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]);
         list_add_tail(&rq->queuelist, &cfqq->fifo);
@@ -3558,7 +3252,7 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd)
 
 static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 {
-        struct cfq_io_context *cic = cfqd->active_cic;
+        struct cfq_io_cq *cic = cfqd->active_cic;
 
         /* If the queue already has requests, don't wait */
         if (!RB_EMPTY_ROOT(&cfqq->sort_list))
@@ -3695,7 +3389,7 @@ static int cfq_may_queue(struct request_queue *q, int rw)
 {
         struct cfq_data *cfqd = q->elevator->elevator_data;
         struct task_struct *tsk = current;
-        struct cfq_io_context *cic;
+        struct cfq_io_cq *cic;
         struct cfq_queue *cfqq;
 
         /*
@@ -3710,7 +3404,7 @@ static int cfq_may_queue(struct request_queue *q, int rw)
 
         cfqq = cic_to_cfqq(cic, rw_is_sync(rw));
         if (cfqq) {
-                cfq_init_prio_data(cfqq, cic->ioc);
+                cfq_init_prio_data(cfqq, cic->icq.ioc);
 
                 return __cfq_may_queue(cfqq);
         }
@@ -3731,21 +3425,17 @@ static void cfq_put_request(struct request *rq)
                 BUG_ON(!cfqq->allocated[rw]);
                 cfqq->allocated[rw]--;
 
-                put_io_context(RQ_CIC(rq)->ioc);
-
-                rq->elevator_private[0] = NULL;
-                rq->elevator_private[1] = NULL;
-
                 /* Put down rq reference on cfqg */
                 cfq_put_cfqg(RQ_CFQG(rq));
-                rq->elevator_private[2] = NULL;
+                rq->elv.priv[0] = NULL;
+                rq->elv.priv[1] = NULL;
 
                 cfq_put_queue(cfqq);
         }
 }
 
 static struct cfq_queue *
-cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_context *cic,
+cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_cq *cic,
                 struct cfq_queue *cfqq)
 {
         cfq_log_cfqq(cfqd, cfqq, "merging with queue %p", cfqq->new_cfqq);
@@ -3760,7 +3450,7 @@ cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_context *cic,
  * was the last process referring to said cfqq.
  */
 static struct cfq_queue *
-split_cfqq(struct cfq_io_context *cic, struct cfq_queue *cfqq)
+split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)
 {
         if (cfqq_process_refs(cfqq) == 1) {
                 cfqq->pid = current->pid;
@@ -3783,25 +3473,29 @@ static int
 cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
 {
         struct cfq_data *cfqd = q->elevator->elevator_data;
-        struct cfq_io_context *cic;
+        struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);
         const int rw = rq_data_dir(rq);
         const bool is_sync = rq_is_sync(rq);
         struct cfq_queue *cfqq;
-        unsigned long flags;
 
         might_sleep_if(gfp_mask & __GFP_WAIT);
 
-        cic = cfq_get_io_context(cfqd, gfp_mask);
-
-        spin_lock_irqsave(q->queue_lock, flags);
+        spin_lock_irq(q->queue_lock);
 
-        if (!cic)
-                goto queue_fail;
+        /* handle changed notifications */
+        if (unlikely(cic->icq.changed)) {
+                if (test_and_clear_bit(ICQ_IOPRIO_CHANGED, &cic->icq.changed))
+                        changed_ioprio(cic);
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
+                if (test_and_clear_bit(ICQ_CGROUP_CHANGED, &cic->icq.changed))
+                        changed_cgroup(cic);
+#endif
+        }
 
 new_queue:
         cfqq = cic_to_cfqq(cic, is_sync);
         if (!cfqq || cfqq == &cfqd->oom_cfqq) {
-                cfqq = cfq_get_queue(cfqd, is_sync, cic->ioc, gfp_mask);
+                cfqq = cfq_get_queue(cfqd, is_sync, cic->icq.ioc, gfp_mask);
                 cic_set_cfqq(cic, cfqq, is_sync);
         } else {
                 /*
@@ -3827,17 +3521,10 @@ new_queue:
         cfqq->allocated[rw]++;
 
         cfqq->ref++;
-        rq->elevator_private[0] = cic;
-        rq->elevator_private[1] = cfqq;
-        rq->elevator_private[2] = cfq_ref_get_cfqg(cfqq->cfqg);
-        spin_unlock_irqrestore(q->queue_lock, flags);
+        rq->elv.priv[0] = cfqq;
+        rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg);
+        spin_unlock_irq(q->queue_lock);
         return 0;
-
-queue_fail:
-        cfq_schedule_dispatch(cfqd);
-        spin_unlock_irqrestore(q->queue_lock, flags);
-        cfq_log(cfqd, "set_request fail");
-        return 1;
 }
 
 static void cfq_kick_queue(struct work_struct *work)
@@ -3941,14 +3628,6 @@ static void cfq_exit_queue(struct elevator_queue *e)
         if (cfqd->active_queue)
                 __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
 
-        while (!list_empty(&cfqd->cic_list)) {
-                struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
-                                                        struct cfq_io_context,
-                                                        queue_list);
-
-                __cfq_exit_single_io_context(cfqd, cic);
-        }
-
         cfq_put_async_queues(cfqd);
         cfq_release_cfq_groups(cfqd);
 
@@ -3963,10 +3642,6 @@ static void cfq_exit_queue(struct elevator_queue *e)
 
         cfq_shutdown_timer_wq(cfqd);
 
-        spin_lock(&cic_index_lock);
-        ida_remove(&cic_index_ida, cfqd->cic_index);
-        spin_unlock(&cic_index_lock);
-
         /*
          * Wait for cfqg->blkg->key accessors to exit their grace periods.
          * Do this wait only if there are other unlinked groups out
@@ -3988,24 +3663,6 @@ static void cfq_exit_queue(struct elevator_queue *e)
         kfree(cfqd);
 }
 
-static int cfq_alloc_cic_index(void)
-{
-        int index, error;
-
-        do {
-                if (!ida_pre_get(&cic_index_ida, GFP_KERNEL))
-                        return -ENOMEM;
-
-                spin_lock(&cic_index_lock);
-                error = ida_get_new(&cic_index_ida, &index);
-                spin_unlock(&cic_index_lock);
-                if (error && error != -EAGAIN)
-                        return error;
-        } while (error);
-
-        return index;
-}
-
 static void *cfq_init_queue(struct request_queue *q)
 {
         struct cfq_data *cfqd;
@@ -4013,23 +3670,9 @@ static void *cfq_init_queue(struct request_queue *q)
         struct cfq_group *cfqg;
         struct cfq_rb_root *st;
 
-        i = cfq_alloc_cic_index();
-        if (i < 0)
-                return NULL;
-
         cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node);
-        if (!cfqd) {
-                spin_lock(&cic_index_lock);
-                ida_remove(&cic_index_ida, i);
-                spin_unlock(&cic_index_lock);
+        if (!cfqd)
                 return NULL;
-        }
-
-        /*
-         * Don't need take queue_lock in the routine, since we are
-         * initializing the ioscheduler, and nobody is using cfqd
-         */
-        cfqd->cic_index = i;
 
         /* Init root service tree */
         cfqd->grp_service_tree = CFQ_RB_ROOT;
@@ -4055,11 +3698,6 @@ static void *cfq_init_queue(struct request_queue *q)
 
         if (blkio_alloc_blkg_stats(&cfqg->blkg)) {
                 kfree(cfqg);
-
-                spin_lock(&cic_index_lock);
-                ida_remove(&cic_index_ida, cfqd->cic_index);
-                spin_unlock(&cic_index_lock);
-
                 kfree(cfqd);
                 return NULL;
         }
@@ -4091,8 +3729,6 @@ static void *cfq_init_queue(struct request_queue *q)
         cfqd->oom_cfqq.ref++;
         cfq_link_cfqq_cfqg(&cfqd->oom_cfqq, &cfqd->root_group);
 
-        INIT_LIST_HEAD(&cfqd->cic_list);
-
         cfqd->queue = q;
 
         init_timer(&cfqd->idle_slice_timer);
@@ -4121,34 +3757,6 @@ static void *cfq_init_queue(struct request_queue *q)
         return cfqd;
 }
 
-static void cfq_slab_kill(void)
-{
-        /*
-         * Caller already ensured that pending RCU callbacks are completed,
-         * so we should have no busy allocations at this point.
-         */
-        if (cfq_pool)
-                kmem_cache_destroy(cfq_pool);
-        if (cfq_ioc_pool)
-                kmem_cache_destroy(cfq_ioc_pool);
-}
-
-static int __init cfq_slab_setup(void)
-{
-        cfq_pool = KMEM_CACHE(cfq_queue, 0);
-        if (!cfq_pool)
-                goto fail;
-
-        cfq_ioc_pool = KMEM_CACHE(cfq_io_context, 0);
-        if (!cfq_ioc_pool)
-                goto fail;
-
-        return 0;
-fail:
-        cfq_slab_kill();
-        return -ENOMEM;
-}
-
 /*
  * sysfs parts below -->
  */
@@ -4254,15 +3862,18 @@ static struct elevator_type iosched_cfq = {
                 .elevator_completed_req_fn =    cfq_completed_request,
                 .elevator_former_req_fn =       elv_rb_former_request,
                 .elevator_latter_req_fn =       elv_rb_latter_request,
+                .elevator_init_icq_fn =         cfq_init_icq,
+                .elevator_exit_icq_fn =         cfq_exit_icq,
                 .elevator_set_req_fn =          cfq_set_request,
                 .elevator_put_req_fn =          cfq_put_request,
                 .elevator_may_queue_fn =        cfq_may_queue,
                 .elevator_init_fn =             cfq_init_queue,
                 .elevator_exit_fn =             cfq_exit_queue,
-                .trim =                         cfq_free_io_context,
         },
+        .icq_size       =       sizeof(struct cfq_io_cq),
+        .icq_align      =       __alignof__(struct cfq_io_cq),
         .elevator_attrs =       cfq_attrs,
-        .elevator_name =        "cfq",
+        .elevator_name  =       "cfq",
         .elevator_owner =       THIS_MODULE,
 };
 
@@ -4280,6 +3891,8 @@ static struct blkio_policy_type blkio_policy_cfq;
 
 static int __init cfq_init(void)
 {
+        int ret;
+
         /*
          * could be 0 on HZ < 1000 setups
          */
@@ -4294,10 +3907,16 @@ static int __init cfq_init(void)
 #else
                 cfq_group_idle = 0;
 #endif
-        if (cfq_slab_setup())
+        cfq_pool = KMEM_CACHE(cfq_queue, 0);
+        if (!cfq_pool)
                 return -ENOMEM;
 
-        elv_register(&iosched_cfq);
+        ret = elv_register(&iosched_cfq);
+        if (ret) {
+                kmem_cache_destroy(cfq_pool);
+                return ret;
+        }
+
         blkio_policy_register(&blkio_policy_cfq);
 
         return 0;
@@ -4305,21 +3924,9 @@ static int __init cfq_init(void)
 
 static void __exit cfq_exit(void)
 {
-        DECLARE_COMPLETION_ONSTACK(all_gone);
         blkio_policy_unregister(&blkio_policy_cfq);
         elv_unregister(&iosched_cfq);
-        ioc_gone = &all_gone;
-        /* ioc_gone's update must be visible before reading ioc_count */
-        smp_wmb();
-
-        /*
-         * this also protects us from entering cfq_slab_kill() with
-         * pending RCU callbacks
-         */
-        if (elv_ioc_count_read(cfq_ioc_count))
-                wait_for_completion(&all_gone);
-        ida_destroy(&cic_index_ida);
-        cfq_slab_kill();
+        kmem_cache_destroy(cfq_pool);
 }
 
 module_init(cfq_init);
diff --git a/block/compat_ioctl.c b/block/compat_ioctl.c
index 7b725020823..7c668c8a6f9 100644
--- a/block/compat_ioctl.c
+++ b/block/compat_ioctl.c
@@ -719,6 +719,9 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg)
         case BLKSECTGET:
                 return compat_put_ushort(arg,
                                          queue_max_sectors(bdev_get_queue(bdev)));
+        case BLKROTATIONAL:
+                return compat_put_ushort(arg,
+                                         !blk_queue_nonrot(bdev_get_queue(bdev)));
         case BLKRASET: /* compatible, but no compat_ptr (!) */
         case BLKFRASET:
                 if (!capable(CAP_SYS_ADMIN))
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c
index c644137d9cd..7bf12d793fc 100644
--- a/block/deadline-iosched.c
+++ b/block/deadline-iosched.c
@@ -448,9 +448,7 @@ static struct elevator_type iosched_deadline = {
 
 static int __init deadline_init(void)
 {
-        elv_register(&iosched_deadline);
-
-        return 0;
+        return elv_register(&iosched_deadline);
 }
 
 static void __exit deadline_exit(void)
diff --git a/block/elevator.c b/block/elevator.c
index 66343d6917d..91e18f8af9b 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -61,8 +61,8 @@ static int elv_iosched_allow_merge(struct request *rq, struct bio *bio)
         struct request_queue *q = rq->q;
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_allow_merge_fn)
-                return e->ops->elevator_allow_merge_fn(q, rq, bio);
+        if (e->type->ops.elevator_allow_merge_fn)
+                return e->type->ops.elevator_allow_merge_fn(q, rq, bio);
 
         return 1;
 }
@@ -168,17 +168,13 @@ static struct elevator_type *elevator_get(const char *name)
         return e;
 }
 
-static void *elevator_init_queue(struct request_queue *q,
-                                 struct elevator_queue *eq)
+static int elevator_init_queue(struct request_queue *q,
+                               struct elevator_queue *eq)
 {
-        return eq->ops->elevator_init_fn(q);
-}
-
-static void elevator_attach(struct request_queue *q, struct elevator_queue *eq,
-                           void *data)
-{
-        q->elevator = eq;
-        eq->elevator_data = data;
+        eq->elevator_data = eq->type->ops.elevator_init_fn(q);
+        if (eq->elevator_data)
+                return 0;
+        return -ENOMEM;
 }
 
 static char chosen_elevator[ELV_NAME_MAX];
@@ -207,8 +203,7 @@ static struct elevator_queue *elevator_alloc(struct request_queue *q,
         if (unlikely(!eq))
                 goto err;
 
-        eq->ops = &e->ops;
-        eq->elevator_type = e;
+        eq->type = e;
         kobject_init(&eq->kobj, &elv_ktype);
         mutex_init(&eq->sysfs_lock);
 
@@ -232,7 +227,7 @@ static void elevator_release(struct kobject *kobj)
         struct elevator_queue *e;
 
         e = container_of(kobj, struct elevator_queue, kobj);
-        elevator_put(e->elevator_type);
+        elevator_put(e->type);
         kfree(e->hash);
         kfree(e);
 }
@@ -241,7 +236,7 @@ int elevator_init(struct request_queue *q, char *name)
 {
         struct elevator_type *e = NULL;
         struct elevator_queue *eq;
-        void *data;
+        int err;
 
         if (unlikely(q->elevator))
                 return 0;
@@ -278,13 +273,13 @@ int elevator_init(struct request_queue *q, char *name)
         if (!eq)
                 return -ENOMEM;
 
-        data = elevator_init_queue(q, eq);
-        if (!data) {
+        err = elevator_init_queue(q, eq);
+        if (err) {
                 kobject_put(&eq->kobj);
-                return -ENOMEM;
+                return err;
         }
 
-        elevator_attach(q, eq, data);
+        q->elevator = eq;
         return 0;
 }
 EXPORT_SYMBOL(elevator_init);
@@ -292,9 +287,8 @@ EXPORT_SYMBOL(elevator_init);
 void elevator_exit(struct elevator_queue *e)
 {
         mutex_lock(&e->sysfs_lock);
-        if (e->ops->elevator_exit_fn)
-                e->ops->elevator_exit_fn(e);
-        e->ops = NULL;
+        if (e->type->ops.elevator_exit_fn)
+                e->type->ops.elevator_exit_fn(e);
         mutex_unlock(&e->sysfs_lock);
 
         kobject_put(&e->kobj);
@@ -504,8 +498,8 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio)
                 return ELEVATOR_BACK_MERGE;
         }
 
-        if (e->ops->elevator_merge_fn)
-                return e->ops->elevator_merge_fn(q, req, bio);
+        if (e->type->ops.elevator_merge_fn)
+                return e->type->ops.elevator_merge_fn(q, req, bio);
 
         return ELEVATOR_NO_MERGE;
 }
@@ -548,8 +542,8 @@ void elv_merged_request(struct request_queue *q, struct request *rq, int type)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_merged_fn)
-                e->ops->elevator_merged_fn(q, rq, type);
+        if (e->type->ops.elevator_merged_fn)
+                e->type->ops.elevator_merged_fn(q, rq, type);
 
         if (type == ELEVATOR_BACK_MERGE)
                 elv_rqhash_reposition(q, rq);
@@ -563,8 +557,8 @@ void elv_merge_requests(struct request_queue *q, struct request *rq,
         struct elevator_queue *e = q->elevator;
         const int next_sorted = next->cmd_flags & REQ_SORTED;
 
-        if (next_sorted && e->ops->elevator_merge_req_fn)
-                e->ops->elevator_merge_req_fn(q, rq, next);
+        if (next_sorted && e->type->ops.elevator_merge_req_fn)
+                e->type->ops.elevator_merge_req_fn(q, rq, next);
 
         elv_rqhash_reposition(q, rq);
 
@@ -581,8 +575,8 @@ void elv_bio_merged(struct request_queue *q, struct request *rq,
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_bio_merged_fn)
-                e->ops->elevator_bio_merged_fn(q, rq, bio);
+        if (e->type->ops.elevator_bio_merged_fn)
+                e->type->ops.elevator_bio_merged_fn(q, rq, bio);
 }
 
 void elv_requeue_request(struct request_queue *q, struct request *rq)
@@ -608,12 +602,12 @@ void elv_drain_elevator(struct request_queue *q)
 
         lockdep_assert_held(q->queue_lock);
 
-        while (q->elevator->ops->elevator_dispatch_fn(q, 1))
+        while (q->elevator->type->ops.elevator_dispatch_fn(q, 1))
                 ;
         if (q->nr_sorted && printed++ < 10) {
                 printk(KERN_ERR "%s: forced dispatching is broken "
                        "(nr_sorted=%u), please report this\n",
-                       q->elevator->elevator_type->elevator_name, q->nr_sorted);
+                       q->elevator->type->elevator_name, q->nr_sorted);
         }
 }
 
@@ -702,7 +696,7 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where)
                  * rq cannot be accessed after calling
                  * elevator_add_req_fn.
                  */
-                q->elevator->ops->elevator_add_req_fn(q, rq);
+                q->elevator->type->ops.elevator_add_req_fn(q, rq);
                 break;
 
         case ELEVATOR_INSERT_FLUSH:
@@ -731,8 +725,8 @@ struct request *elv_latter_request(struct request_queue *q, struct request *rq)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_latter_req_fn)
-                return e->ops->elevator_latter_req_fn(q, rq);
+        if (e->type->ops.elevator_latter_req_fn)
+                return e->type->ops.elevator_latter_req_fn(q, rq);
         return NULL;
 }
 
@@ -740,8 +734,8 @@ struct request *elv_former_request(struct request_queue *q, struct request *rq)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_former_req_fn)
-                return e->ops->elevator_former_req_fn(q, rq);
+        if (e->type->ops.elevator_former_req_fn)
+                return e->type->ops.elevator_former_req_fn(q, rq);
         return NULL;
 }
 
@@ -749,10 +743,8 @@ int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_set_req_fn)
-                return e->ops->elevator_set_req_fn(q, rq, gfp_mask);
-
-        rq->elevator_private[0] = NULL;
+        if (e->type->ops.elevator_set_req_fn)
+                return e->type->ops.elevator_set_req_fn(q, rq, gfp_mask);
         return 0;
 }
 
@@ -760,16 +752,16 @@ void elv_put_request(struct request_queue *q, struct request *rq)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_put_req_fn)
-                e->ops->elevator_put_req_fn(rq);
+        if (e->type->ops.elevator_put_req_fn)
+                e->type->ops.elevator_put_req_fn(rq);
 }
 
 int elv_may_queue(struct request_queue *q, int rw)
 {
         struct elevator_queue *e = q->elevator;
 
-        if (e->ops->elevator_may_queue_fn)
-                return e->ops->elevator_may_queue_fn(q, rw);
+        if (e->type->ops.elevator_may_queue_fn)
+                return e->type->ops.elevator_may_queue_fn(q, rw);
 
         return ELV_MQUEUE_MAY;
 }
@@ -804,8 +796,8 @@ void elv_completed_request(struct request_queue *q, struct request *rq)
         if (blk_account_rq(rq)) {
                 q->in_flight[rq_is_sync(rq)]--;
                 if ((rq->cmd_flags & REQ_SORTED) &&
-                    e->ops->elevator_completed_req_fn)
-                        e->ops->elevator_completed_req_fn(q, rq);
+                    e->type->ops.elevator_completed_req_fn)
+                        e->type->ops.elevator_completed_req_fn(q, rq);
         }
 }
 
@@ -823,7 +815,7 @@ elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 
         e = container_of(kobj, struct elevator_queue, kobj);
         mutex_lock(&e->sysfs_lock);
-        error = e->ops ? entry->show(e, page) : -ENOENT;
+        error = e->type ? entry->show(e, page) : -ENOENT;
         mutex_unlock(&e->sysfs_lock);
         return error;
 }
@@ -841,7 +833,7 @@ elv_attr_store(struct kobject *kobj, struct attribute *attr,
 
         e = container_of(kobj, struct elevator_queue, kobj);
         mutex_lock(&e->sysfs_lock);
-        error = e->ops ? entry->store(e, page, length) : -ENOENT;
+        error = e->type ? entry->store(e, page, length) : -ENOENT;
         mutex_unlock(&e->sysfs_lock);
         return error;
 }
@@ -856,14 +848,13 @@ static struct kobj_type elv_ktype = {
         .release        = elevator_release,
 };
 
-int elv_register_queue(struct request_queue *q)
+int __elv_register_queue(struct request_queue *q, struct elevator_queue *e)
 {
-        struct elevator_queue *e = q->elevator;
         int error;
 
         error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
         if (!error) {
-                struct elv_fs_entry *attr = e->elevator_type->elevator_attrs;
+                struct elv_fs_entry *attr = e->type->elevator_attrs;
                 if (attr) {
                         while (attr->attr.name) {
                                 if (sysfs_create_file(&e->kobj, &attr->attr))
@@ -876,31 +867,55 @@ int elv_register_queue(struct request_queue *q)
         }
         return error;
 }
-EXPORT_SYMBOL(elv_register_queue);
 
-static void __elv_unregister_queue(struct elevator_queue *e)
+int elv_register_queue(struct request_queue *q)
 {
-        kobject_uevent(&e->kobj, KOBJ_REMOVE);
-        kobject_del(&e->kobj);
-        e->registered = 0;
+        return __elv_register_queue(q, q->elevator);
 }
+EXPORT_SYMBOL(elv_register_queue);
 
 void elv_unregister_queue(struct request_queue *q)
 {
-        if (q)
-                __elv_unregister_queue(q->elevator);
+        if (q) {
+                struct elevator_queue *e = q->elevator;
+
+                kobject_uevent(&e->kobj, KOBJ_REMOVE);
+                kobject_del(&e->kobj);
+                e->registered = 0;
+        }
 }
 EXPORT_SYMBOL(elv_unregister_queue);
 
-void elv_register(struct elevator_type *e)
+int elv_register(struct elevator_type *e)
 {
         char *def = "";
 
+        /* create icq_cache if requested */
+        if (e->icq_size) {
+                if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
+                    WARN_ON(e->icq_align < __alignof__(struct io_cq)))
+                        return -EINVAL;
+
+                snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
+                         "%s_io_cq", e->elevator_name);
+                e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
+                                                 e->icq_align, 0, NULL);
+                if (!e->icq_cache)
+                        return -ENOMEM;
+        }
+
+        /* register, don't allow duplicate names */
         spin_lock(&elv_list_lock);
-        BUG_ON(elevator_find(e->elevator_name));
+        if (elevator_find(e->elevator_name)) {
+                spin_unlock(&elv_list_lock);
+                if (e->icq_cache)
+                        kmem_cache_destroy(e->icq_cache);
+                return -EBUSY;
+        }
         list_add_tail(&e->list, &elv_list);
         spin_unlock(&elv_list_lock);
 
+        /* print pretty message */
         if (!strcmp(e->elevator_name, chosen_elevator) ||
                         (!*chosen_elevator &&
                          !strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
@@ -908,30 +923,26 @@ void elv_register(struct elevator_type *e)
 
         printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,
                                                                 def);
+        return 0;
 }
 EXPORT_SYMBOL_GPL(elv_register);
 
 void elv_unregister(struct elevator_type *e)
 {
-        struct task_struct *g, *p;
+        /* unregister */
+        spin_lock(&elv_list_lock);
+        list_del_init(&e->list);
+        spin_unlock(&elv_list_lock);
 
         /*
-         * Iterate every thread in the process to remove the io contexts.
+         * Destroy icq_cache if it exists.  icq's are RCU managed.  Make
+         * sure all RCU operations are complete before proceeding.
          */
-        if (e->ops.trim) {
-                read_lock(&tasklist_lock);
-                do_each_thread(g, p) {
-                        task_lock(p);
-                        if (p->io_context)
-                                e->ops.trim(p->io_context);
-                        task_unlock(p);
-                } while_each_thread(g, p);
-                read_unlock(&tasklist_lock);
+        if (e->icq_cache) {
+                rcu_barrier();
+                kmem_cache_destroy(e->icq_cache);
+                e->icq_cache = NULL;
         }
-
-        spin_lock(&elv_list_lock);
-        list_del_init(&e->list);
-        spin_unlock(&elv_list_lock);
 }
 EXPORT_SYMBOL_GPL(elv_unregister);
 
@@ -944,54 +955,41 @@ EXPORT_SYMBOL_GPL(elv_unregister);
 static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
 {
         struct elevator_queue *old_elevator, *e;
-        void *data;
         int err;
 
-        /*
-         * Allocate new elevator
-         */
+        /* allocate new elevator */
         e = elevator_alloc(q, new_e);
         if (!e)
                 return -ENOMEM;
 
-        data = elevator_init_queue(q, e);
-        if (!data) {
+        err = elevator_init_queue(q, e);
+        if (err) {
                 kobject_put(&e->kobj);
-                return -ENOMEM;
+                return err;
         }
 
-        /*
-         * Turn on BYPASS and drain all requests w/ elevator private data
-         */
+        /* turn on BYPASS and drain all requests w/ elevator private data */
         elv_quiesce_start(q);
 
-        /*
-         * Remember old elevator.
-         */
-        old_elevator = q->elevator;
-
-        /*
-         * attach and start new elevator
-         */
-        spin_lock_irq(q->queue_lock);
-        elevator_attach(q, e, data);
-        spin_unlock_irq(q->queue_lock);
-
-        if (old_elevator->registered) {
-                __elv_unregister_queue(old_elevator);
-
-                err = elv_register_queue(q);
+        /* unregister old queue, register new one and kill old elevator */
+        if (q->elevator->registered) {
+                elv_unregister_queue(q);
+                err = __elv_register_queue(q, e);
                 if (err)
                         goto fail_register;
         }
 
-        /*
-         * finally exit old elevator and turn off BYPASS.
-         */
+        /* done, clear io_cq's, switch elevators and turn off BYPASS */
+        spin_lock_irq(q->queue_lock);
+        ioc_clear_queue(q);
+        old_elevator = q->elevator;
+        q->elevator = e;
+        spin_unlock_irq(q->queue_lock);
+
         elevator_exit(old_elevator);
         elv_quiesce_end(q);
 
-        blk_add_trace_msg(q, "elv switch: %s", e->elevator_type->elevator_name);
+        blk_add_trace_msg(q, "elv switch: %s", e->type->elevator_name);
 
         return 0;
 
@@ -1001,7 +999,6 @@ fail_register:
          * one again (along with re-adding the sysfs dir)
          */
         elevator_exit(e);
-        q->elevator = old_elevator;
         elv_register_queue(q);
         elv_quiesce_end(q);
 
@@ -1026,7 +1023,7 @@ int elevator_change(struct request_queue *q, const char *name)
                 return -EINVAL;
         }
 
-        if (!strcmp(elevator_name, q->elevator->elevator_type->elevator_name)) {
+        if (!strcmp(elevator_name, q->elevator->type->elevator_name)) {
                 elevator_put(e);
                 return 0;
         }
@@ -1061,7 +1058,7 @@ ssize_t elv_iosched_show(struct request_queue *q, char *name)
         if (!q->elevator || !blk_queue_stackable(q))
                 return sprintf(name, "none\n");
 
-        elv = e->elevator_type;
+        elv = e->type;
 
         spin_lock(&elv_list_lock);
         list_for_each_entry(__e, &elv_list, list) {
diff --git a/block/genhd.c b/block/genhd.c
index 83e7c04015e..23b4f706332 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -614,7 +614,7 @@ void add_disk(struct gendisk *disk)
          * Take an extra ref on queue which will be put on disk_release()
          * so that it sticks around as long as @disk is there.
          */
-        WARN_ON_ONCE(blk_get_queue(disk->queue));
+        WARN_ON_ONCE(!blk_get_queue(disk->queue));
 
         retval = sysfs_create_link(&disk_to_dev(disk)->kobj, &bdi->dev->kobj,
                                    "bdi");
diff --git a/block/ioctl.c b/block/ioctl.c
index 4828fa34981..ba15b2dbfb9 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -296,6 +296,8 @@ int blkdev_ioctl(struct block_device *bdev, fmode_t mode, unsigned cmd,
                 return put_uint(arg, bdev_discard_zeroes_data(bdev));
         case BLKSECTGET:
                 return put_ushort(arg, queue_max_sectors(bdev_get_queue(bdev)));
+        case BLKROTATIONAL:
+                return put_ushort(arg, !blk_queue_nonrot(bdev_get_queue(bdev)));
         case BLKRASET:
         case BLKFRASET:
                 if(!capable(CAP_SYS_ADMIN))
diff --git a/block/noop-iosched.c b/block/noop-iosched.c
index 06389e9ef96..413a0b1d788 100644
--- a/block/noop-iosched.c
+++ b/block/noop-iosched.c
@@ -94,9 +94,7 @@ static struct elevator_type elevator_noop = {
 
 static int __init noop_init(void)
 {
-        elv_register(&elevator_noop);
-
-        return 0;
+        return elv_register(&elevator_noop);
 }
 
 static void __exit noop_exit(void)