1 files changed, 64 insertions, 17 deletions

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 5f127cfb2e92..ed897b5ef315 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -55,6 +55,7 @@ static const int cfq_hist_divisor = 4;
 #define RQ_CIC(rq)              \
         ((struct cfq_io_context *) (rq)->elevator_private)
 #define RQ_CFQQ(rq)             (struct cfq_queue *) ((rq)->elevator_private2)
+#define RQ_CFQG(rq)             (struct cfq_group *) ((rq)->elevator_private3)
 
 static struct kmem_cache *cfq_pool;
 static struct kmem_cache *cfq_ioc_pool;
@@ -143,8 +144,6 @@ struct cfq_queue {
         struct cfq_queue *new_cfqq;
         struct cfq_group *cfqg;
         struct cfq_group *orig_cfqg;
-        /* Sectors dispatched in current dispatch round */
-        unsigned long nr_sectors;
 };
 
 /*
@@ -346,7 +345,7 @@ CFQ_CFQQ_FNS(deep);
 CFQ_CFQQ_FNS(wait_busy);
 #undef CFQ_CFQQ_FNS
 
-#ifdef CONFIG_DEBUG_CFQ_IOSCHED
+#ifdef CONFIG_CFQ_GROUP_IOSCHED
 #define cfq_log_cfqq(cfqd, cfqq, fmt, args...)  \
         blk_add_trace_msg((cfqd)->queue, "cfq%d%c %s " fmt, (cfqq)->pid, \
                         cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
@@ -858,7 +857,7 @@ cfq_group_service_tree_del(struct cfq_data *cfqd, struct cfq_group *cfqg)
         if (!RB_EMPTY_NODE(&cfqg->rb_node))
                 cfq_rb_erase(&cfqg->rb_node, st);
         cfqg->saved_workload_slice = 0;
-        blkiocg_update_blkio_group_dequeue_stats(&cfqg->blkg, 1);
+        blkiocg_update_dequeue_stats(&cfqg->blkg, 1);
 }
 
 static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq)
@@ -884,8 +883,7 @@ static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq)
                         slice_used = cfqq->allocated_slice;
         }
 
-        cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u sect=%lu", slice_used,
-                                cfqq->nr_sectors);
+        cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u", slice_used);
         return slice_used;
 }
 
@@ -919,8 +917,8 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
 
         cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,
                                         st->min_vdisktime);
-        blkiocg_update_blkio_group_stats(&cfqg->blkg, used_sl,
-                                                cfqq->nr_sectors);
+        blkiocg_update_timeslice_used(&cfqg->blkg, used_sl);
+        blkiocg_set_start_empty_time(&cfqg->blkg);
 }
 
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
@@ -961,7 +959,6 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
         if (!cfqg)
                 goto done;
 
-        cfqg->weight = blkcg->weight;
         for_each_cfqg_st(cfqg, i, j, st)
                 *st = CFQ_RB_ROOT;
         RB_CLEAR_NODE(&cfqg->rb_node);
@@ -978,6 +975,7 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
         sscanf(dev_name(bdi->dev), "%u:%u", &major, &minor);
         blkiocg_add_blkio_group(blkcg, &cfqg->blkg, (void *)cfqd,
                                         MKDEV(major, minor));
+        cfqg->weight = blkcg_get_weight(blkcg, cfqg->blkg.dev);
 
         /* Add group on cfqd list */
         hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list);
@@ -1004,6 +1002,12 @@ static struct cfq_group *cfq_get_cfqg(struct cfq_data *cfqd, int create)
         return cfqg;
 }
 
+static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
+{
+        atomic_inc(&cfqg->ref);
+        return cfqg;
+}
+
 static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)
 {
         /* Currently, all async queues are mapped to root group */
@@ -1087,6 +1091,12 @@ static struct cfq_group *cfq_get_cfqg(struct cfq_data *cfqd, int create)
 {
         return &cfqd->root_group;
 }
+
+static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
+{
+        return cfqg;
+}
+
 static inline void
 cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {
         cfqq->cfqg = cfqg;
@@ -1389,7 +1399,12 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)
 {
         elv_rb_del(&cfqq->sort_list, rq);
         cfqq->queued[rq_is_sync(rq)]--;
+        blkiocg_update_io_remove_stats(&(RQ_CFQG(rq))->blkg, rq_data_dir(rq),
+                                                rq_is_sync(rq));
         cfq_add_rq_rb(rq);
+        blkiocg_update_io_add_stats(&(RQ_CFQG(rq))->blkg,
+                        &cfqq->cfqd->serving_group->blkg, rq_data_dir(rq),
+                        rq_is_sync(rq));
 }
 
 static struct request *
@@ -1445,6 +1460,8 @@ static void cfq_remove_request(struct request *rq)
         cfq_del_rq_rb(rq);
 
         cfqq->cfqd->rq_queued--;
+        blkiocg_update_io_remove_stats(&(RQ_CFQG(rq))->blkg, rq_data_dir(rq),
+                                                rq_is_sync(rq));
         if (rq_is_meta(rq)) {
                 WARN_ON(!cfqq->meta_pending);
                 cfqq->meta_pending--;
@@ -1476,6 +1493,13 @@ static void cfq_merged_request(struct request_queue *q, struct request *req,
         }
 }
 
+static void cfq_bio_merged(struct request_queue *q, struct request *req,
+                                struct bio *bio)
+{
+        blkiocg_update_io_merged_stats(&(RQ_CFQG(req))->blkg, bio_data_dir(bio),
+                                        cfq_bio_sync(bio));
+}
+
 static void
 cfq_merged_requests(struct request_queue *q, struct request *rq,
                     struct request *next)
@@ -1493,6 +1517,8 @@ cfq_merged_requests(struct request_queue *q, struct request *rq,
         if (cfqq->next_rq == next)
                 cfqq->next_rq = rq;
         cfq_remove_request(next);
+        blkiocg_update_io_merged_stats(&(RQ_CFQG(rq))->blkg, rq_data_dir(next),
+                                        rq_is_sync(next));
 }
 
 static int cfq_allow_merge(struct request_queue *q, struct request *rq,
@@ -1520,18 +1546,24 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq,
         return cfqq == RQ_CFQQ(rq);
 }
 
+static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
+{
+        del_timer(&cfqd->idle_slice_timer);
+        blkiocg_update_idle_time_stats(&cfqq->cfqg->blkg);
+}
+
 static void __cfq_set_active_queue(struct cfq_data *cfqd,
                                    struct cfq_queue *cfqq)
 {
         if (cfqq) {
                 cfq_log_cfqq(cfqd, cfqq, "set_active wl_prio:%d wl_type:%d",
                                 cfqd->serving_prio, cfqd->serving_type);
+                blkiocg_update_avg_queue_size_stats(&cfqq->cfqg->blkg);
                 cfqq->slice_start = 0;
                 cfqq->dispatch_start = jiffies;
                 cfqq->allocated_slice = 0;
                 cfqq->slice_end = 0;
                 cfqq->slice_dispatch = 0;
-                cfqq->nr_sectors = 0;
 
                 cfq_clear_cfqq_wait_request(cfqq);
                 cfq_clear_cfqq_must_dispatch(cfqq);
@@ -1539,7 +1571,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd,
                 cfq_clear_cfqq_fifo_expire(cfqq);
                 cfq_mark_cfqq_slice_new(cfqq);
 
-                del_timer(&cfqd->idle_slice_timer);
+                cfq_del_timer(cfqd, cfqq);
         }
 
         cfqd->active_queue = cfqq;
@@ -1555,7 +1587,7 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
         cfq_log_cfqq(cfqd, cfqq, "slice expired t=%d", timed_out);
 
         if (cfq_cfqq_wait_request(cfqq))
-                del_timer(&cfqd->idle_slice_timer);
+                cfq_del_timer(cfqd, cfqq);
 
         cfq_clear_cfqq_wait_request(cfqq);
         cfq_clear_cfqq_wait_busy(cfqq);
@@ -1857,6 +1889,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
         sl = cfqd->cfq_slice_idle;
 
         mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
+        blkiocg_update_set_idle_time_stats(&cfqq->cfqg->blkg);
         cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu", sl);
 }
 
@@ -1876,7 +1909,8 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq)
         elv_dispatch_sort(q, rq);
 
         cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
-        cfqq->nr_sectors += blk_rq_sectors(rq);
+        blkiocg_update_dispatch_stats(&cfqq->cfqg->blkg, blk_rq_bytes(rq),
+                                        rq_data_dir(rq), rq_is_sync(rq));
 }
 
 /*
@@ -3185,11 +3219,14 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                 if (cfq_cfqq_wait_request(cfqq)) {
                         if (blk_rq_bytes(rq) > PAGE_CACHE_SIZE ||
                             cfqd->busy_queues > 1) {
-                                del_timer(&cfqd->idle_slice_timer);
+                                cfq_del_timer(cfqd, cfqq);
                                 cfq_clear_cfqq_wait_request(cfqq);
                                 __blk_run_queue(cfqd->queue);
-                        } else
+                        } else {
+                                blkiocg_update_idle_time_stats(
+                                                &cfqq->cfqg->blkg);
                                 cfq_mark_cfqq_must_dispatch(cfqq);
+                        }
                 }
         } else if (cfq_should_preempt(cfqd, cfqq, rq)) {
                 /*
@@ -3214,7 +3251,9 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)
         rq_set_fifo_time(rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]);
         list_add_tail(&rq->queuelist, &cfqq->fifo);
         cfq_add_rq_rb(rq);
-
+        blkiocg_update_io_add_stats(&(RQ_CFQG(rq))->blkg,
+                        &cfqd->serving_group->blkg, rq_data_dir(rq),
+                        rq_is_sync(rq));
         cfq_rq_enqueued(cfqd, cfqq, rq);
 }
 
@@ -3300,6 +3339,9 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
         WARN_ON(!cfqq->dispatched);
         cfqd->rq_in_driver--;
         cfqq->dispatched--;
+        blkiocg_update_completion_stats(&cfqq->cfqg->blkg, rq_start_time_ns(rq),
+                        rq_io_start_time_ns(rq), rq_data_dir(rq),
+                        rq_is_sync(rq));
 
         cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
 
@@ -3440,6 +3482,10 @@ static void cfq_put_request(struct request *rq)
                 rq->elevator_private = NULL;
                 rq->elevator_private2 = NULL;
 
+                /* Put down rq reference on cfqg */
+                cfq_put_cfqg(RQ_CFQG(rq));
+                rq->elevator_private3 = NULL;
+
                 cfq_put_queue(cfqq);
         }
 }
@@ -3528,6 +3574,7 @@ new_queue:
 
         rq->elevator_private = cic;
         rq->elevator_private2 = cfqq;
+        rq->elevator_private3 = cfq_ref_get_cfqg(cfqq->cfqg);
         return 0;
 
 queue_fail:
@@ -3743,7 +3790,6 @@ static void *cfq_init_queue(struct request_queue *q)
          * second, in order to have larger depth for async operations.
          */
         cfqd->last_delayed_sync = jiffies - HZ;
-        INIT_RCU_HEAD(&cfqd->rcu);
         return cfqd;
 }
 
@@ -3872,6 +3918,7 @@ static struct elevator_type iosched_cfq = {
                 .elevator_merged_fn =           cfq_merged_request,
                 .elevator_merge_req_fn =        cfq_merged_requests,
                 .elevator_allow_merge_fn =      cfq_allow_merge,
+                .elevator_bio_merged_fn =       cfq_bio_merged,
                 .elevator_dispatch_fn =         cfq_dispatch_requests,
                 .elevator_add_req_fn =          cfq_insert_request,
                 .elevator_activate_req_fn =     cfq_activate_request,