1 files changed, 45 insertions, 23 deletions

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 420eaa150d11..a8d23f0cf357 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -44,6 +44,7 @@ struct throtl_service_queue {
         struct rb_node          *first_pending; /* first node in the tree */
         unsigned int            nr_pending;     /* # queued in the tree */
         unsigned long           first_pending_disptime; /* disptime of the first tg */
+        struct timer_list       pending_timer;  /* fires on first_pending_disptime */
 };
 
 enum tg_state_flags {
@@ -121,7 +122,7 @@ struct throtl_data
         unsigned int nr_undestroyed_grps;
 
         /* Work for dispatching throttled bios */
-        struct delayed_work dispatch_work;
+        struct work_struct dispatch_work;
 };
 
 /* list and work item to allocate percpu group stats */
@@ -131,6 +132,8 @@ static LIST_HEAD(tg_stats_alloc_list);
 static void tg_stats_alloc_fn(struct work_struct *);
 static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
 
+static void throtl_pending_timer_fn(unsigned long arg);
+
 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
 {
         return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
@@ -255,6 +258,13 @@ static void throtl_service_queue_init(struct throtl_service_queue *sq,
         bio_list_init(&sq->bio_lists[1]);
         sq->pending_tree = RB_ROOT;
         sq->parent_sq = parent_sq;
+        setup_timer(&sq->pending_timer, throtl_pending_timer_fn,
+                    (unsigned long)sq);
+}
+
+static void throtl_service_queue_exit(struct throtl_service_queue *sq)
+{
+        del_timer_sync(&sq->pending_timer);
 }
 
 static void throtl_pd_init(struct blkcg_gq *blkg)
@@ -293,6 +303,8 @@ static void throtl_pd_exit(struct blkcg_gq *blkg)
         spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
 
         free_percpu(tg->stats_cpu);
+
+        throtl_service_queue_exit(&tg->service_queue);
 }
 
 static void throtl_pd_reset_stats(struct blkcg_gq *blkg)
@@ -447,19 +459,17 @@ static void throtl_dequeue_tg(struct throtl_grp *tg)
 }
 
 /* Call with queue lock held */
-static void throtl_schedule_delayed_work(struct throtl_data *td,
-                                         unsigned long delay)
+static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,
+                                          unsigned long expires)
 {
-        struct delayed_work *dwork = &td->dispatch_work;
-        struct throtl_service_queue *sq = &td->service_queue;
-
-        mod_delayed_work(kthrotld_workqueue, dwork, delay);
-        throtl_log(sq, "schedule work. delay=%lu jiffies=%lu", delay, jiffies);
+        mod_timer(&sq->pending_timer, expires);
+        throtl_log(sq, "schedule timer. delay=%lu jiffies=%lu",
+                   expires - jiffies, jiffies);
 }
 
-static void throtl_schedule_next_dispatch(struct throtl_data *td)
+static void throtl_schedule_next_dispatch(struct throtl_service_queue *sq)
 {
-        struct throtl_service_queue *sq = &td->service_queue;
+        struct throtl_data *td = sq_to_td(sq);
 
         /* any pending children left? */
         if (!sq->nr_pending)
@@ -467,10 +477,14 @@ static void throtl_schedule_next_dispatch(struct throtl_data *td)
 
         update_min_dispatch_time(sq);
 
-        if (time_before_eq(sq->first_pending_disptime, jiffies))
-                throtl_schedule_delayed_work(td, 0);
-        else
-                throtl_schedule_delayed_work(td, sq->first_pending_disptime - jiffies);
+        /* is the next dispatch time in the future? */
+        if (time_after(sq->first_pending_disptime, jiffies)) {
+                throtl_schedule_pending_timer(sq, sq->first_pending_disptime);
+                return;
+        }
+
+        /* kick immediate execution */
+        queue_work(kthrotld_workqueue, &td->dispatch_work);
 }
 
 static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)
@@ -901,11 +915,19 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
         return nr_disp;
 }
 
+static void throtl_pending_timer_fn(unsigned long arg)
+{
+        struct throtl_service_queue *sq = (void *)arg;
+        struct throtl_data *td = sq_to_td(sq);
+
+        queue_work(kthrotld_workqueue, &td->dispatch_work);
+}
+
 /* work function to dispatch throttled bios */
 void blk_throtl_dispatch_work_fn(struct work_struct *work)
 {
-        struct throtl_data *td = container_of(to_delayed_work(work),
-                                              struct throtl_data, dispatch_work);
+        struct throtl_data *td = container_of(work, struct throtl_data,
+                                              dispatch_work);
         struct throtl_service_queue *sq = &td->service_queue;
         struct request_queue *q = td->queue;
         unsigned int nr_disp = 0;
@@ -932,7 +954,7 @@ void blk_throtl_dispatch_work_fn(struct work_struct *work)
                 throtl_log(sq, "bios disp=%u", nr_disp);
         }
 
-        throtl_schedule_next_dispatch(td);
+        throtl_schedule_next_dispatch(sq);
 
         spin_unlock_irq(q->queue_lock);
 
@@ -1020,7 +1042,7 @@ static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
         struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
         struct blkg_conf_ctx ctx;
         struct throtl_grp *tg;
-        struct throtl_data *td;
+        struct throtl_service_queue *sq;
         int ret;
 
         ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
@@ -1028,7 +1050,7 @@ static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
                 return ret;
 
         tg = blkg_to_tg(ctx.blkg);
-        td = ctx.blkg->q->td;
+        sq = &tg->service_queue;
 
         if (!ctx.v)
                 ctx.v = -1;
@@ -1056,7 +1078,7 @@ static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
 
         if (tg->flags & THROTL_TG_PENDING) {
                 tg_update_disptime(tg);
-                throtl_schedule_next_dispatch(td);
+                throtl_schedule_next_dispatch(sq->parent_sq);
         }
 
         blkg_conf_finish(&ctx);
@@ -1121,7 +1143,7 @@ static void throtl_shutdown_wq(struct request_queue *q)
 {
         struct throtl_data *td = q->td;
 
-        cancel_delayed_work_sync(&td->dispatch_work);
+        cancel_work_sync(&td->dispatch_work);
 }
 
 static struct blkcg_policy blkcg_policy_throtl = {
@@ -1210,7 +1232,7 @@ queue_bio:
         /* update @tg's dispatch time if @tg was empty before @bio */
         if (tg->flags & THROTL_TG_WAS_EMPTY) {
                 tg_update_disptime(tg);
-                throtl_schedule_next_dispatch(td);
+                throtl_schedule_next_dispatch(tg->service_queue.parent_sq);
         }
 
 out_unlock:
@@ -1273,7 +1295,7 @@ int blk_throtl_init(struct request_queue *q)
         if (!td)
                 return -ENOMEM;
 
-        INIT_DELAYED_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn);
+        INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn);
         throtl_service_queue_init(&td->service_queue, NULL);
 
         q->td = td;