9 files changed, 243 insertions, 75 deletions

diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 31610ae0ebff..9f97da52d006 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -9,6 +9,10 @@
  *
  * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  *                    Nauman Rafique <nauman@google.com>
+ *
+ * For policy-specific per-blkcg data:
+ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
+ *                    Arianna Avanzini <avanzini.arianna@gmail.com>
  */
 #include <linux/ioprio.h>
 #include <linux/kdev_t.h>
@@ -27,8 +31,7 @@
 
 static DEFINE_MUTEX(blkcg_pol_mutex);
 
-struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT,
-                            .cfq_leaf_weight = 2 * CFQ_WEIGHT_DEFAULT, };
+struct blkcg blkcg_root;
 EXPORT_SYMBOL_GPL(blkcg_root);
 
 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
@@ -827,6 +830,8 @@ static struct cgroup_subsys_state *
 blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
 {
         struct blkcg *blkcg;
+        struct cgroup_subsys_state *ret;
+        int i;
 
         if (!parent_css) {
                 blkcg = &blkcg_root;
@@ -834,11 +839,35 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
         }
 
         blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
-        if (!blkcg)
-                return ERR_PTR(-ENOMEM);
+        if (!blkcg) {
+                ret = ERR_PTR(-ENOMEM);
+                goto free_blkcg;
+        }
+
+        for (i = 0; i < BLKCG_MAX_POLS ; i++) {
+                struct blkcg_policy *pol = blkcg_policy[i];
+                struct blkcg_policy_data *cpd;
+
+                /*
+                 * If the policy hasn't been attached yet, wait for it
+                 * to be attached before doing anything else. Otherwise,
+                 * check if the policy requires any specific per-cgroup
+                 * data: if it does, allocate and initialize it.
+                 */
+                if (!pol || !pol->cpd_size)
+                        continue;
+
+                BUG_ON(blkcg->pd[i]);
+                cpd = kzalloc(pol->cpd_size, GFP_KERNEL);
+                if (!cpd) {
+                        ret = ERR_PTR(-ENOMEM);
+                        goto free_pd_blkcg;
+                }
+                blkcg->pd[i] = cpd;
+                cpd->plid = i;
+                pol->cpd_init_fn(blkcg);
+        }
 
-        blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
-        blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT;
 done:
         spin_lock_init(&blkcg->lock);
         INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
@@ -847,6 +876,14 @@ done:
         INIT_LIST_HEAD(&blkcg->cgwb_list);
 #endif
         return &blkcg->css;
+
+free_pd_blkcg:
+        for (i--; i >= 0; i--)
+                kfree(blkcg->pd[i]);
+
+free_blkcg:
+        kfree(blkcg);
+        return ret;
 }
 
 /**
@@ -1000,8 +1037,10 @@ int blkcg_activate_policy(struct request_queue *q,
                           const struct blkcg_policy *pol)
 {
         LIST_HEAD(pds);
+        LIST_HEAD(cpds);
         struct blkcg_gq *blkg;
-        struct blkg_policy_data *pd, *n;
+        struct blkg_policy_data *pd, *nd;
+        struct blkcg_policy_data *cpd, *cnd;
         int cnt = 0, ret;
 
         if (blkcg_policy_enabled(q, pol))
@@ -1014,6 +1053,10 @@ int blkcg_activate_policy(struct request_queue *q,
                 cnt++;
         spin_unlock_irq(q->queue_lock);
 
+        /*
+         * Allocate per-blkg and per-blkcg policy data
+         * for all existing blkgs.
+         */
         while (cnt--) {
                 pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
                 if (!pd) {
@@ -1021,26 +1064,50 @@ int blkcg_activate_policy(struct request_queue *q,
                         goto out_free;
                 }
                 list_add_tail(&pd->alloc_node, &pds);
+
+                if (!pol->cpd_size)
+                        continue;
+                cpd = kzalloc_node(pol->cpd_size, GFP_KERNEL, q->node);
+                if (!cpd) {
+                        ret = -ENOMEM;
+                        goto out_free;
+                }
+                list_add_tail(&cpd->alloc_node, &cpds);
         }
 
         /*
-         * Install the allocated pds.  With @q bypassing, no new blkg
+         * Install the allocated pds and cpds. With @q bypassing, no new blkg
          * should have been created while the queue lock was dropped.
          */
         spin_lock_irq(q->queue_lock);
 
         list_for_each_entry(blkg, &q->blkg_list, q_node) {
-                if (WARN_ON(list_empty(&pds))) {
+                if (WARN_ON(list_empty(&pds)) ||
+                    WARN_ON(pol->cpd_size && list_empty(&cpds))) {
                         /* umm... this shouldn't happen, just abort */
                         ret = -ENOMEM;
                         goto out_unlock;
                 }
+                cpd = list_first_entry(&cpds, struct blkcg_policy_data,
+                                       alloc_node);
+                list_del_init(&cpd->alloc_node);
                 pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node);
                 list_del_init(&pd->alloc_node);
 
                 /* grab blkcg lock too while installing @pd on @blkg */
                 spin_lock(&blkg->blkcg->lock);
 
+                if (!pol->cpd_size)
+                        goto no_cpd;
+                if (!blkg->blkcg->pd[pol->plid]) {
+                        /* Per-policy per-blkcg data */
+                        blkg->blkcg->pd[pol->plid] = cpd;
+                        cpd->plid = pol->plid;
+                        pol->cpd_init_fn(blkg->blkcg);
+                } else { /* must free it as it has already been extracted */
+                        kfree(cpd);
+                }
+no_cpd:
                 blkg->pd[pol->plid] = pd;
                 pd->blkg = blkg;
                 pd->plid = pol->plid;
@@ -1055,8 +1122,10 @@ out_unlock:
         spin_unlock_irq(q->queue_lock);
 out_free:
         blk_queue_bypass_end(q);
-        list_for_each_entry_safe(pd, n, &pds, alloc_node)
+        list_for_each_entry_safe(pd, nd, &pds, alloc_node)
                 kfree(pd);
+        list_for_each_entry_safe(cpd, cnd, &cpds, alloc_node)
+                kfree(cpd);
         return ret;
 }
 EXPORT_SYMBOL_GPL(blkcg_activate_policy);
@@ -1093,6 +1162,8 @@ void blkcg_deactivate_policy(struct request_queue *q,
 
                 kfree(blkg->pd[pol->plid]);
                 blkg->pd[pol->plid] = NULL;
+                kfree(blkg->blkcg->pd[pol->plid]);
+                blkg->blkcg->pd[pol->plid] = NULL;
 
                 spin_unlock(&blkg->blkcg->lock);
         }
diff --git a/block/blk-core.c b/block/blk-core.c
index a4a2dbe46fe3..688ae9482cb8 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -579,6 +579,8 @@ void blk_cleanup_queue(struct request_queue *q)
                 q->queue_lock = &q->__queue_lock;
         spin_unlock_irq(lock);
 
+        bdi_destroy(&q->backing_dev_info);
+
         /* @q is and will stay empty, shutdown and put */
         blk_put_queue(q);
 }
@@ -758,6 +760,8 @@ blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
 }
 EXPORT_SYMBOL(blk_init_queue_node);
 
+static void blk_queue_bio(struct request_queue *q, struct bio *bio);
+
 struct request_queue *
 blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
                          spinlock_t *lock)
@@ -1601,7 +1605,7 @@ void init_request_from_bio(struct request *req, struct bio *bio)
         blk_rq_bio_prep(req->q, req, bio);
 }
 
-void blk_queue_bio(struct request_queue *q, struct bio *bio)
+static void blk_queue_bio(struct request_queue *q, struct bio *bio)
 {
         const bool sync = !!(bio->bi_rw & REQ_SYNC);
         struct blk_plug *plug;
@@ -1709,7 +1713,6 @@ out_unlock:
                 spin_unlock_irq(q->queue_lock);
         }
 }
-EXPORT_SYMBOL_GPL(blk_queue_bio);       /* for device mapper only */
 
 /*
  * If bio->bi_dev is a partition, remap the location
diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c
index 5f13f4d0bcce..1e28ddb656b8 100644
--- a/block/blk-mq-cpumap.c
+++ b/block/blk-mq-cpumap.c
@@ -24,7 +24,7 @@ static int get_first_sibling(unsigned int cpu)
 {
         unsigned int ret;
 
-        ret = cpumask_first(topology_thread_cpumask(cpu));
+        ret = cpumask_first(topology_sibling_cpumask(cpu));
         if (ret < nr_cpu_ids)
                 return ret;
 
diff --git a/block/blk-mq.c b/block/blk-mq.c
index ef100fd2cb86..f53779692c77 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -673,8 +673,11 @@ static void blk_mq_rq_timer(unsigned long priv)
                 data.next = blk_rq_timeout(round_jiffies_up(data.next));
                 mod_timer(&q->timeout, data.next);
         } else {
-                queue_for_each_hw_ctx(q, hctx, i)
-                        blk_mq_tag_idle(hctx);
+                queue_for_each_hw_ctx(q, hctx, i) {
+                        /* the hctx may be unmapped, so check it here */
+                        if (blk_mq_hw_queue_mapped(hctx))
+                                blk_mq_tag_idle(hctx);
+                }
         }
 }
 
@@ -851,6 +854,16 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
                 spin_lock(&hctx->lock);
                 list_splice(&rq_list, &hctx->dispatch);
                 spin_unlock(&hctx->lock);
+                /*
+                 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
+                 * it's possible the queue is stopped and restarted again
+                 * before this. Queue restart will dispatch requests. And since
+                 * requests in rq_list aren't added into hctx->dispatch yet,
+                 * the requests in rq_list might get lost.
+                 *
+                 * blk_mq_run_hw_queue() already checks the STOPPED bit
+                 **/
+                blk_mq_run_hw_queue(hctx, true);
         }
 }
 
@@ -1600,22 +1613,6 @@ static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
         return NOTIFY_OK;
 }
 
-static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
-{
-        struct request_queue *q = hctx->queue;
-        struct blk_mq_tag_set *set = q->tag_set;
-
-        if (set->tags[hctx->queue_num])
-                return NOTIFY_OK;
-
-        set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
-        if (!set->tags[hctx->queue_num])
-                return NOTIFY_STOP;
-
-        hctx->tags = set->tags[hctx->queue_num];
-        return NOTIFY_OK;
-}
-
 static int blk_mq_hctx_notify(void *data, unsigned long action,
                               unsigned int cpu)
 {
@@ -1623,12 +1620,16 @@ static int blk_mq_hctx_notify(void *data, unsigned long action,
 
         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
                 return blk_mq_hctx_cpu_offline(hctx, cpu);
-        else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
-                return blk_mq_hctx_cpu_online(hctx, cpu);
+
+        /*
+         * In case of CPU online, tags may be reallocated
+         * in blk_mq_map_swqueue() after mapping is updated.
+         */
 
         return NOTIFY_OK;
 }
 
+/* hctx->ctxs will be freed in queue's release handler */
 static void blk_mq_exit_hctx(struct request_queue *q,
                 struct blk_mq_tag_set *set,
                 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
@@ -1647,7 +1648,6 @@ static void blk_mq_exit_hctx(struct request_queue *q,
 
         blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
         blk_free_flush_queue(hctx->fq);
-        kfree(hctx->ctxs);
         blk_mq_free_bitmap(&hctx->ctx_map);
 }
 
@@ -1804,6 +1804,7 @@ static void blk_mq_map_swqueue(struct request_queue *q)
         unsigned int i;
         struct blk_mq_hw_ctx *hctx;
         struct blk_mq_ctx *ctx;
+        struct blk_mq_tag_set *set = q->tag_set;
 
         queue_for_each_hw_ctx(q, hctx, i) {
                 cpumask_clear(hctx->cpumask);
@@ -1833,16 +1834,20 @@ static void blk_mq_map_swqueue(struct request_queue *q)
                  * disable it and free the request entries.
                  */
                 if (!hctx->nr_ctx) {
-                        struct blk_mq_tag_set *set = q->tag_set;
-
                         if (set->tags[i]) {
                                 blk_mq_free_rq_map(set, set->tags[i], i);
                                 set->tags[i] = NULL;
-                                hctx->tags = NULL;
                         }
+                        hctx->tags = NULL;
                         continue;
                 }
 
+                /* unmapped hw queue can be remapped after CPU topo changed */
+                if (!set->tags[i])
+                        set->tags[i] = blk_mq_init_rq_map(set, i);
+                hctx->tags = set->tags[i];
+                WARN_ON(!hctx->tags);
+
                 /*
                  * Set the map size to the number of mapped software queues.
                  * This is more accurate and more efficient than looping
@@ -1916,8 +1921,12 @@ void blk_mq_release(struct request_queue *q)
         unsigned int i;
 
         /* hctx kobj stays in hctx */
-        queue_for_each_hw_ctx(q, hctx, i)
+        queue_for_each_hw_ctx(q, hctx, i) {
+                if (!hctx)
+                        continue;
+                kfree(hctx->ctxs);
                 kfree(hctx);
+        }
 
         kfree(q->queue_hw_ctx);
 
@@ -2120,9 +2129,16 @@ static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
          */
         list_for_each_entry(q, &all_q_list, all_q_node)
                 blk_mq_freeze_queue_start(q);
-        list_for_each_entry(q, &all_q_list, all_q_node)
+        list_for_each_entry(q, &all_q_list, all_q_node) {
                 blk_mq_freeze_queue_wait(q);
 
+                /*
+                 * timeout handler can't touch hw queue during the
+                 * reinitialization
+                 */
+                del_timer_sync(&q->timeout);
+        }
+
         list_for_each_entry(q, &all_q_list, all_q_node)
                 blk_mq_queue_reinit(q);
 
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 1b60941dc4c6..6264b382d4d1 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -523,8 +523,6 @@ static void blk_release_queue(struct kobject *kobj)
 
         blk_trace_shutdown(q);
 
-        bdi_destroy(&q->backing_dev_info);
-
         ida_simple_remove(&blk_queue_ida, q->id);
         call_rcu(&q->rcu_head, blk_free_queue_rcu);
 }
diff --git a/block/bounce.c b/block/bounce.c
index 072280b3dd13..b17311227c12 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -219,8 +219,8 @@ bounce:
                 if (page_to_pfn(page) <= queue_bounce_pfn(q) && !force)
                         continue;
 
-                inc_zone_page_state(to->bv_page, NR_BOUNCE);
                 to->bv_page = mempool_alloc(pool, q->bounce_gfp);
+                inc_zone_page_state(to->bv_page, NR_BOUNCE);
 
                 if (rw == WRITE) {
                         char *vto, *vfrom;
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index bc8f42930773..c62bb2e650b8 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -67,6 +67,11 @@ static struct kmem_cache *cfq_pool;
 #define sample_valid(samples)   ((samples) > 80)
 #define rb_entry_cfqg(node)     rb_entry((node), struct cfq_group, rb_node)
 
+/* blkio-related constants */
+#define CFQ_WEIGHT_MIN          10
+#define CFQ_WEIGHT_MAX          1000
+#define CFQ_WEIGHT_DEFAULT      500
+
 struct cfq_ttime {
         unsigned long last_end_request;
 
@@ -212,6 +217,15 @@ struct cfqg_stats {
 #endif  /* CONFIG_CFQ_GROUP_IOSCHED */
 };
 
+/* Per-cgroup data */
+struct cfq_group_data {
+        /* must be the first member */
+        struct blkcg_policy_data pd;
+
+        unsigned int weight;
+        unsigned int leaf_weight;
+};
+
 /* This is per cgroup per device grouping structure */
 struct cfq_group {
         /* must be the first member */
@@ -446,16 +460,6 @@ CFQ_CFQQ_FNS(deep);
 CFQ_CFQQ_FNS(wait_busy);
 #undef CFQ_CFQQ_FNS
 
-static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
-{
-        return pd ? container_of(pd, struct cfq_group, pd) : NULL;
-}
-
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
-{
-        return pd_to_blkg(&cfqg->pd);
-}
-
 #if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
 
 /* cfqg stats flags */
@@ -600,6 +604,22 @@ static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
 
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
 
+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
+{
+        return pd ? container_of(pd, struct cfq_group, pd) : NULL;
+}
+
+static struct cfq_group_data
+*cpd_to_cfqgd(struct blkcg_policy_data *cpd)
+{
+        return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL;
+}
+
+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
+{
+        return pd_to_blkg(&cfqg->pd);
+}
+
 static struct blkcg_policy blkcg_policy_cfq;
 
 static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
@@ -607,6 +627,11 @@ static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
         return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
 }
 
+static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg)
+{
+        return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq));
+}
+
 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)
 {
         struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;
@@ -1544,13 +1569,28 @@ static void cfqg_stats_init(struct cfqg_stats *stats)
 #endif
 }
 
+static void cfq_cpd_init(const struct blkcg *blkcg)
+{
+        struct cfq_group_data *cgd =
+                cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]);
+
+        if (blkcg == &blkcg_root) {
+                cgd->weight = 2 * CFQ_WEIGHT_DEFAULT;
+                cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT;
+        } else {
+                cgd->weight = CFQ_WEIGHT_DEFAULT;
+                cgd->leaf_weight = CFQ_WEIGHT_DEFAULT;
+        }
+}
+
 static void cfq_pd_init(struct blkcg_gq *blkg)
 {
         struct cfq_group *cfqg = blkg_to_cfqg(blkg);
+        struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg);
 
         cfq_init_cfqg_base(cfqg);
-        cfqg->weight = blkg->blkcg->cfq_weight;
-        cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight;
+        cfqg->weight = cgd->weight;
+        cfqg->leaf_weight = cgd->leaf_weight;
         cfqg_stats_init(&cfqg->stats);
         cfqg_stats_init(&cfqg->dead_stats);
 }
@@ -1673,13 +1713,27 @@ static int cfqg_print_leaf_weight_device(struct seq_file *sf, void *v)
 
 static int cfq_print_weight(struct seq_file *sf, void *v)
 {
-        seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_weight);
+        struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+        struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+        unsigned int val = 0;
+
+        if (cgd)
+                val = cgd->weight;
+
+        seq_printf(sf, "%u\n", val);
         return 0;
 }
 
 static int cfq_print_leaf_weight(struct seq_file *sf, void *v)
 {
-        seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_leaf_weight);
+        struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+        struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+        unsigned int val = 0;
+
+        if (cgd)
+                val = cgd->leaf_weight;
+
+        seq_printf(sf, "%u\n", val);
         return 0;
 }
 
@@ -1690,6 +1744,7 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
         struct blkcg *blkcg = css_to_blkcg(of_css(of));
         struct blkg_conf_ctx ctx;
         struct cfq_group *cfqg;
+        struct cfq_group_data *cfqgd;
         int ret;
 
         ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
@@ -1698,17 +1753,22 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
 
         ret = -EINVAL;
         cfqg = blkg_to_cfqg(ctx.blkg);
+        cfqgd = blkcg_to_cfqgd(blkcg);
+        if (!cfqg || !cfqgd)
+                goto err;
+
         if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) {
                 if (!is_leaf_weight) {
                         cfqg->dev_weight = ctx.v;
-                        cfqg->new_weight = ctx.v ?: blkcg->cfq_weight;
+                        cfqg->new_weight = ctx.v ?: cfqgd->weight;
                 } else {
                         cfqg->dev_leaf_weight = ctx.v;
-                        cfqg->new_leaf_weight = ctx.v ?: blkcg->cfq_leaf_weight;
+                        cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight;
                 }
                 ret = 0;
         }
 
+err:
         blkg_conf_finish(&ctx);
         return ret ?: nbytes;
 }
@@ -1730,16 +1790,23 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
 {
         struct blkcg *blkcg = css_to_blkcg(css);
         struct blkcg_gq *blkg;
+        struct cfq_group_data *cfqgd;
+        int ret = 0;
 
         if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
                 return -EINVAL;
 
         spin_lock_irq(&blkcg->lock);
+        cfqgd = blkcg_to_cfqgd(blkcg);
+        if (!cfqgd) {
+                ret = -EINVAL;
+                goto out;
+        }
 
         if (!is_leaf_weight)
-                blkcg->cfq_weight = val;
+                cfqgd->weight = val;
         else
-                blkcg->cfq_leaf_weight = val;
+                cfqgd->leaf_weight = val;
 
         hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
                 struct cfq_group *cfqg = blkg_to_cfqg(blkg);
@@ -1749,15 +1816,16 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
 
                 if (!is_leaf_weight) {
                         if (!cfqg->dev_weight)
-                                cfqg->new_weight = blkcg->cfq_weight;
+                                cfqg->new_weight = cfqgd->weight;
                 } else {
                         if (!cfqg->dev_leaf_weight)
-                                cfqg->new_leaf_weight = blkcg->cfq_leaf_weight;
+                                cfqg->new_leaf_weight = cfqgd->leaf_weight;
                 }
         }
 
+out:
         spin_unlock_irq(&blkcg->lock);
-        return 0;
+        return ret;
 }
 
 static int cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
@@ -4477,6 +4545,18 @@ out_free:
         return ret;
 }
 
+static void cfq_registered_queue(struct request_queue *q)
+{
+        struct elevator_queue *e = q->elevator;
+        struct cfq_data *cfqd = e->elevator_data;
+
+        /*
+         * Default to IOPS mode with no idling for SSDs
+         */
+        if (blk_queue_nonrot(q))
+                cfqd->cfq_slice_idle = 0;
+}
+
 /*
  * sysfs parts below -->
  */
@@ -4592,6 +4672,7 @@ static struct elevator_type iosched_cfq = {
                 .elevator_may_queue_fn =        cfq_may_queue,
                 .elevator_init_fn =             cfq_init_queue,
                 .elevator_exit_fn =             cfq_exit_queue,
+                .elevator_registered_fn =       cfq_registered_queue,
         },
         .icq_size       =       sizeof(struct cfq_io_cq),
         .icq_align      =       __alignof__(struct cfq_io_cq),
@@ -4603,8 +4684,10 @@ static struct elevator_type iosched_cfq = {
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
 static struct blkcg_policy blkcg_policy_cfq = {
         .pd_size                = sizeof(struct cfq_group),
+        .cpd_size               = sizeof(struct cfq_group_data),
         .cftypes                = cfq_blkcg_files,
 
+        .cpd_init_fn            = cfq_cpd_init,
         .pd_init_fn             = cfq_pd_init,
         .pd_offline_fn          = cfq_pd_offline,
         .pd_reset_stats_fn      = cfq_pd_reset_stats,
diff --git a/block/elevator.c b/block/elevator.c
index 3bbb48f430e4..84d63943f2de 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -157,7 +157,7 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
 
         eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
         if (unlikely(!eq))
-                goto err;
+                return NULL;
 
         eq->type = e;
         kobject_init(&eq->kobj, &elv_ktype);
@@ -165,10 +165,6 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
         hash_init(eq->hash);
 
         return eq;
-err:
-        kfree(eq);
-        elevator_put(e);
-        return NULL;
 }
 EXPORT_SYMBOL(elevator_alloc);
 
@@ -810,6 +806,8 @@ int elv_register_queue(struct request_queue *q)
                 }
                 kobject_uevent(&e->kobj, KOBJ_ADD);
                 e->registered = 1;
+                if (e->type->ops.elevator_registered_fn)
+                        e->type->ops.elevator_registered_fn(q);
         }
         return error;
 }
diff --git a/block/genhd.c b/block/genhd.c
index d46ba566d62f..59a1395eedac 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -423,9 +423,9 @@ int blk_alloc_devt(struct hd_struct *part, dev_t *devt)
         /* allocate ext devt */
         idr_preload(GFP_KERNEL);
 
-        spin_lock(&ext_devt_lock);
+        spin_lock_bh(&ext_devt_lock);
         idx = idr_alloc(&ext_devt_idr, part, 0, NR_EXT_DEVT, GFP_NOWAIT);
-        spin_unlock(&ext_devt_lock);
+        spin_unlock_bh(&ext_devt_lock);
 
         idr_preload_end();
         if (idx < 0)
@@ -450,9 +450,9 @@ void blk_free_devt(dev_t devt)
                 return;
 
         if (MAJOR(devt) == BLOCK_EXT_MAJOR) {
-                spin_lock(&ext_devt_lock);
+                spin_lock_bh(&ext_devt_lock);
                 idr_remove(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
-                spin_unlock(&ext_devt_lock);
+                spin_unlock_bh(&ext_devt_lock);
         }
 }
 
@@ -654,7 +654,6 @@ void del_gendisk(struct gendisk *disk)
         disk->flags &= ~GENHD_FL_UP;
 
         sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");
-        bdi_unregister(&disk->queue->backing_dev_info);
         blk_unregister_queue(disk);
         blk_unregister_region(disk_devt(disk), disk->minors);
 
@@ -692,13 +691,13 @@ struct gendisk *get_gendisk(dev_t devt, int *partno)
         } else {
                 struct hd_struct *part;
 
-                spin_lock(&ext_devt_lock);
+                spin_lock_bh(&ext_devt_lock);
                 part = idr_find(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
                 if (part && get_disk(part_to_disk(part))) {
                         *partno = part->partno;
                         disk = part_to_disk(part);
                 }
-                spin_unlock(&ext_devt_lock);
+                spin_unlock_bh(&ext_devt_lock);
         }
 
         return disk;