1 files changed, 104 insertions, 21 deletions

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 5da8e6e9ab4b..d8ad45ccd8fa 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -67,6 +67,11 @@ static struct kmem_cache *cfq_pool;
 #define sample_valid(samples)   ((samples) > 80)
 #define rb_entry_cfqg(node)     rb_entry((node), struct cfq_group, rb_node)
 
+/* blkio-related constants */
+#define CFQ_WEIGHT_MIN          10
+#define CFQ_WEIGHT_MAX          1000
+#define CFQ_WEIGHT_DEFAULT      500
+
 struct cfq_ttime {
         unsigned long last_end_request;
 
@@ -212,6 +217,15 @@ struct cfqg_stats {
 #endif  /* CONFIG_CFQ_GROUP_IOSCHED */
 };
 
+/* Per-cgroup data */
+struct cfq_group_data {
+        /* must be the first member */
+        struct blkcg_policy_data pd;
+
+        unsigned int weight;
+        unsigned int leaf_weight;
+};
+
 /* This is per cgroup per device grouping structure */
 struct cfq_group {
         /* must be the first member */
@@ -446,16 +460,6 @@ CFQ_CFQQ_FNS(deep);
 CFQ_CFQQ_FNS(wait_busy);
 #undef CFQ_CFQQ_FNS
 
-static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
-{
-        return pd ? container_of(pd, struct cfq_group, pd) : NULL;
-}
-
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
-{
-        return pd_to_blkg(&cfqg->pd);
-}
-
 #if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
 
 /* cfqg stats flags */
@@ -600,6 +604,22 @@ static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
 
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
 
+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
+{
+        return pd ? container_of(pd, struct cfq_group, pd) : NULL;
+}
+
+static struct cfq_group_data
+*cpd_to_cfqgd(struct blkcg_policy_data *cpd)
+{
+        return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL;
+}
+
+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
+{
+        return pd_to_blkg(&cfqg->pd);
+}
+
 static struct blkcg_policy blkcg_policy_cfq;
 
 static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
@@ -607,6 +627,11 @@ static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
         return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
 }
 
+static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg)
+{
+        return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq));
+}
+
 static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)
 {
         struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;
@@ -1544,13 +1569,28 @@ static void cfqg_stats_init(struct cfqg_stats *stats)
 #endif
 }
 
+static void cfq_cpd_init(const struct blkcg *blkcg)
+{
+        struct cfq_group_data *cgd =
+                cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]);
+
+        if (blkcg == &blkcg_root) {
+                cgd->weight = 2 * CFQ_WEIGHT_DEFAULT;
+                cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT;
+        } else {
+                cgd->weight = CFQ_WEIGHT_DEFAULT;
+                cgd->leaf_weight = CFQ_WEIGHT_DEFAULT;
+        }
+}
+
 static void cfq_pd_init(struct blkcg_gq *blkg)
 {
         struct cfq_group *cfqg = blkg_to_cfqg(blkg);
+        struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg);
 
         cfq_init_cfqg_base(cfqg);
-        cfqg->weight = blkg->blkcg->cfq_weight;
-        cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight;
+        cfqg->weight = cgd->weight;
+        cfqg->leaf_weight = cgd->leaf_weight;
         cfqg_stats_init(&cfqg->stats);
         cfqg_stats_init(&cfqg->dead_stats);
 }
@@ -1673,13 +1713,27 @@ static int cfqg_print_leaf_weight_device(struct seq_file *sf, void *v)
 
 static int cfq_print_weight(struct seq_file *sf, void *v)
 {
-        seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_weight);
+        struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+        struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+        unsigned int val = 0;
+
+        if (cgd)
+                val = cgd->weight;
+
+        seq_printf(sf, "%u\n", val);
         return 0;
 }
 
 static int cfq_print_leaf_weight(struct seq_file *sf, void *v)
 {
-        seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_leaf_weight);
+        struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+        struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
+        unsigned int val = 0;
+
+        if (cgd)
+                val = cgd->leaf_weight;
+
+        seq_printf(sf, "%u\n", val);
         return 0;
 }
 
@@ -1690,6 +1744,7 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
         struct blkcg *blkcg = css_to_blkcg(of_css(of));
         struct blkg_conf_ctx ctx;
         struct cfq_group *cfqg;
+        struct cfq_group_data *cfqgd;
         int ret;
 
         ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
@@ -1698,17 +1753,22 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
 
         ret = -EINVAL;
         cfqg = blkg_to_cfqg(ctx.blkg);
+        cfqgd = blkcg_to_cfqgd(blkcg);
+        if (!cfqg || !cfqgd)
+                goto err;
+
         if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) {
                 if (!is_leaf_weight) {
                         cfqg->dev_weight = ctx.v;
-                        cfqg->new_weight = ctx.v ?: blkcg->cfq_weight;
+                        cfqg->new_weight = ctx.v ?: cfqgd->weight;
                 } else {
                         cfqg->dev_leaf_weight = ctx.v;
-                        cfqg->new_leaf_weight = ctx.v ?: blkcg->cfq_leaf_weight;
+                        cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight;
                 }
                 ret = 0;
         }
 
+err:
         blkg_conf_finish(&ctx);
         return ret ?: nbytes;
 }
@@ -1730,16 +1790,23 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
 {
         struct blkcg *blkcg = css_to_blkcg(css);
         struct blkcg_gq *blkg;
+        struct cfq_group_data *cfqgd;
+        int ret = 0;
 
         if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
                 return -EINVAL;
 
         spin_lock_irq(&blkcg->lock);
+        cfqgd = blkcg_to_cfqgd(blkcg);
+        if (!cfqgd) {
+                ret = -EINVAL;
+                goto out;
+        }
 
         if (!is_leaf_weight)
-                blkcg->cfq_weight = val;
+                cfqgd->weight = val;
         else
-                blkcg->cfq_leaf_weight = val;
+                cfqgd->leaf_weight = val;
 
         hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
                 struct cfq_group *cfqg = blkg_to_cfqg(blkg);
@@ -1749,15 +1816,16 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
 
                 if (!is_leaf_weight) {
                         if (!cfqg->dev_weight)
-                                cfqg->new_weight = blkcg->cfq_weight;
+                                cfqg->new_weight = cfqgd->weight;
                 } else {
                         if (!cfqg->dev_leaf_weight)
-                                cfqg->new_leaf_weight = blkcg->cfq_leaf_weight;
+                                cfqg->new_leaf_weight = cfqgd->leaf_weight;
                 }
         }
 
+out:
         spin_unlock_irq(&blkcg->lock);
-        return 0;
+        return ret;
 }
 
 static int cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
@@ -4477,6 +4545,18 @@ out_free:
         return ret;
 }
 
+static void cfq_registered_queue(struct request_queue *q)
+{
+        struct elevator_queue *e = q->elevator;
+        struct cfq_data *cfqd = e->elevator_data;
+
+        /*
+         * Default to IOPS mode with no idling for SSDs
+         */
+        if (blk_queue_nonrot(q))
+                cfqd->cfq_slice_idle = 0;
+}
+
 /*
  * sysfs parts below -->
  */
@@ -4592,6 +4672,7 @@ static struct elevator_type iosched_cfq = {
                 .elevator_may_queue_fn =        cfq_may_queue,
                 .elevator_init_fn =             cfq_init_queue,
                 .elevator_exit_fn =             cfq_exit_queue,
+                .elevator_registered_fn =       cfq_registered_queue,
         },
         .icq_size       =       sizeof(struct cfq_io_cq),
         .icq_align      =       __alignof__(struct cfq_io_cq),
@@ -4603,8 +4684,10 @@ static struct elevator_type iosched_cfq = {
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
 static struct blkcg_policy blkcg_policy_cfq = {
         .pd_size                = sizeof(struct cfq_group),
+        .cpd_size               = sizeof(struct cfq_group_data),
         .cftypes                = cfq_blkcg_files,
 
+        .cpd_init_fn            = cfq_cpd_init,
         .pd_init_fn             = cfq_pd_init,
         .pd_offline_fn          = cfq_pd_offline,
         .pd_reset_stats_fn      = cfq_pd_reset_stats,