1 files changed, 39 insertions, 42 deletions

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index ec9397f3eb0a..00cfdd05f98f 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -27,6 +27,8 @@ static struct blkcg_policy blkcg_policy_throtl;
 static struct workqueue_struct *kthrotld_workqueue;
 
 struct throtl_service_queue {
+        struct throtl_service_queue *parent_sq; /* the parent service_queue */
+
         /*
          * Bios queued directly to this service_queue or dispatched from
          * children throtl_grp's.
@@ -197,21 +199,24 @@ alloc_stats:
 }
 
 /* init a service_queue, assumes the caller zeroed it */
-static void throtl_service_queue_init(struct throtl_service_queue *sq)
+static void throtl_service_queue_init(struct throtl_service_queue *sq,
+                                      struct throtl_service_queue *parent_sq)
 {
         bio_list_init(&sq->bio_lists[0]);
         bio_list_init(&sq->bio_lists[1]);
         sq->pending_tree = RB_ROOT;
+        sq->parent_sq = parent_sq;
 }
 
 static void throtl_pd_init(struct blkcg_gq *blkg)
 {
         struct throtl_grp *tg = blkg_to_tg(blkg);
+        struct throtl_data *td = blkg->q->td;
         unsigned long flags;
 
-        throtl_service_queue_init(&tg->service_queue);
+        throtl_service_queue_init(&tg->service_queue, &td->service_queue);
         RB_CLEAR_NODE(&tg->rb_node);
-        tg->td = blkg->q->td;
+        tg->td = td;
 
         tg->bps[READ] = -1;
         tg->bps[WRITE] = -1;
@@ -339,9 +344,9 @@ static void update_min_dispatch_time(struct throtl_service_queue *parent_sq)
         parent_sq->first_pending_disptime = tg->disptime;
 }
 
-static void tg_service_queue_add(struct throtl_grp *tg,
-                                 struct throtl_service_queue *parent_sq)
+static void tg_service_queue_add(struct throtl_grp *tg)
 {
+        struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq;
         struct rb_node **node = &parent_sq->pending_tree.rb_node;
         struct rb_node *parent = NULL;
         struct throtl_grp *__tg;
@@ -367,33 +372,29 @@ static void tg_service_queue_add(struct throtl_grp *tg,
         rb_insert_color(&tg->rb_node, &parent_sq->pending_tree);
 }
 
-static void __throtl_enqueue_tg(struct throtl_grp *tg,
-                                struct throtl_service_queue *parent_sq)
+static void __throtl_enqueue_tg(struct throtl_grp *tg)
 {
-        tg_service_queue_add(tg, parent_sq);
+        tg_service_queue_add(tg);
         tg->flags |= THROTL_TG_PENDING;
-        parent_sq->nr_pending++;
+        tg->service_queue.parent_sq->nr_pending++;
 }
 
-static void throtl_enqueue_tg(struct throtl_grp *tg,
-                              struct throtl_service_queue *parent_sq)
+static void throtl_enqueue_tg(struct throtl_grp *tg)
 {
         if (!(tg->flags & THROTL_TG_PENDING))
-                __throtl_enqueue_tg(tg, parent_sq);
+                __throtl_enqueue_tg(tg);
 }
 
-static void __throtl_dequeue_tg(struct throtl_grp *tg,
-                                struct throtl_service_queue *parent_sq)
+static void __throtl_dequeue_tg(struct throtl_grp *tg)
 {
-        throtl_rb_erase(&tg->rb_node, parent_sq);
+        throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
         tg->flags &= ~THROTL_TG_PENDING;
 }
 
-static void throtl_dequeue_tg(struct throtl_grp *tg,
-                              struct throtl_service_queue *parent_sq)
+static void throtl_dequeue_tg(struct throtl_grp *tg)
 {
         if (tg->flags & THROTL_TG_PENDING)
-                __throtl_dequeue_tg(tg, parent_sq);
+                __throtl_dequeue_tg(tg);
 }
 
 /* Call with queue lock held */
@@ -707,8 +708,7 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
         throtl_update_dispatch_stats(tg_to_blkg(tg), bio->bi_size, bio->bi_rw);
 }
 
-static void throtl_add_bio_tg(struct bio *bio, struct throtl_grp *tg,
-                              struct throtl_service_queue *parent_sq)
+static void throtl_add_bio_tg(struct bio *bio, struct throtl_grp *tg)
 {
         struct throtl_service_queue *sq = &tg->service_queue;
         bool rw = bio_data_dir(bio);
@@ -727,11 +727,10 @@ static void throtl_add_bio_tg(struct bio *bio, struct throtl_grp *tg,
         blkg_get(tg_to_blkg(tg));
         sq->nr_queued[rw]++;
         tg->td->nr_queued[rw]++;
-        throtl_enqueue_tg(tg, parent_sq);
+        throtl_enqueue_tg(tg);
 }
 
-static void tg_update_disptime(struct throtl_grp *tg,
-                               struct throtl_service_queue *parent_sq)
+static void tg_update_disptime(struct throtl_grp *tg)
 {
         struct throtl_service_queue *sq = &tg->service_queue;
         unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime;
@@ -747,16 +746,15 @@ static void tg_update_disptime(struct throtl_grp *tg,
         disptime = jiffies + min_wait;
 
         /* Update dispatch time */
-        throtl_dequeue_tg(tg, parent_sq);
+        throtl_dequeue_tg(tg);
         tg->disptime = disptime;
-        throtl_enqueue_tg(tg, parent_sq);
+        throtl_enqueue_tg(tg);
 
         /* see throtl_add_bio_tg() */
         tg->flags &= ~THROTL_TG_WAS_EMPTY;
 }
 
-static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw,
-                                struct throtl_service_queue *parent_sq)
+static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)
 {
         struct throtl_service_queue *sq = &tg->service_queue;
         struct bio *bio;
@@ -770,14 +768,13 @@ static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw,
         tg->td->nr_queued[rw]--;
 
         throtl_charge_bio(tg, bio);
-        bio_list_add(&parent_sq->bio_lists[rw], bio);
+        bio_list_add(&sq->parent_sq->bio_lists[rw], bio);
         bio->bi_rw |= REQ_THROTTLED;
 
         throtl_trim_slice(tg, rw);
 }
 
-static int throtl_dispatch_tg(struct throtl_grp *tg,
-                              struct throtl_service_queue *parent_sq)
+static int throtl_dispatch_tg(struct throtl_grp *tg)
 {
         struct throtl_service_queue *sq = &tg->service_queue;
         unsigned int nr_reads = 0, nr_writes = 0;
@@ -790,7 +787,7 @@ static int throtl_dispatch_tg(struct throtl_grp *tg,
         while ((bio = bio_list_peek(&sq->bio_lists[READ])) &&
                tg_may_dispatch(tg, bio, NULL)) {
 
-                tg_dispatch_one_bio(tg, bio_data_dir(bio), parent_sq);
+                tg_dispatch_one_bio(tg, bio_data_dir(bio));
                 nr_reads++;
 
                 if (nr_reads >= max_nr_reads)
@@ -800,7 +797,7 @@ static int throtl_dispatch_tg(struct throtl_grp *tg,
         while ((bio = bio_list_peek(&sq->bio_lists[WRITE])) &&
                tg_may_dispatch(tg, bio, NULL)) {
 
-                tg_dispatch_one_bio(tg, bio_data_dir(bio), parent_sq);
+                tg_dispatch_one_bio(tg, bio_data_dir(bio));
                 nr_writes++;
 
                 if (nr_writes >= max_nr_writes)
@@ -824,12 +821,12 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
                 if (time_before(jiffies, tg->disptime))
                         break;
 
-                throtl_dequeue_tg(tg, parent_sq);
+                throtl_dequeue_tg(tg);
 
-                nr_disp += throtl_dispatch_tg(tg, parent_sq);
+                nr_disp += throtl_dispatch_tg(tg);
 
                 if (sq->nr_queued[0] || sq->nr_queued[1])
-                        tg_update_disptime(tg, parent_sq);
+                        tg_update_disptime(tg);
 
                 if (nr_disp >= throtl_quantum)
                         break;
@@ -991,7 +988,7 @@ static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
         throtl_start_new_slice(tg, 1);
 
         if (tg->flags & THROTL_TG_PENDING) {
-                tg_update_disptime(tg, &td->service_queue);
+                tg_update_disptime(tg);
                 throtl_schedule_next_dispatch(td);
         }
 
@@ -1142,12 +1139,12 @@ queue_bio:
                         sq->nr_queued[READ], sq->nr_queued[WRITE]);
 
         bio_associate_current(bio);
-        throtl_add_bio_tg(bio, tg, &q->td->service_queue);
+        throtl_add_bio_tg(bio, tg);
         throttled = true;
 
         /* update @tg's dispatch time if @tg was empty before @bio */
         if (tg->flags & THROTL_TG_WAS_EMPTY) {
-                tg_update_disptime(tg, &td->service_queue);
+                tg_update_disptime(tg);
                 throtl_schedule_next_dispatch(td);
         }
 
@@ -1179,12 +1176,12 @@ void blk_throtl_drain(struct request_queue *q)
         while ((tg = throtl_rb_first(parent_sq))) {
                 struct throtl_service_queue *sq = &tg->service_queue;
 
-                throtl_dequeue_tg(tg, parent_sq);
+                throtl_dequeue_tg(tg);
 
                 while ((bio = bio_list_peek(&sq->bio_lists[READ])))
-                        tg_dispatch_one_bio(tg, bio_data_dir(bio), parent_sq);
+                        tg_dispatch_one_bio(tg, bio_data_dir(bio));
                 while ((bio = bio_list_peek(&sq->bio_lists[WRITE])))
-                        tg_dispatch_one_bio(tg, bio_data_dir(bio), parent_sq);
+                        tg_dispatch_one_bio(tg, bio_data_dir(bio));
         }
         spin_unlock_irq(q->queue_lock);
 
@@ -1205,7 +1202,7 @@ int blk_throtl_init(struct request_queue *q)
                 return -ENOMEM;
 
         INIT_DELAYED_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn);
-        throtl_service_queue_init(&td->service_queue);
+        throtl_service_queue_init(&td->service_queue, NULL);
 
         q->td = td;
         td->queue = q;