5 files changed, 87 insertions, 29 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 034cbb2024f0..7e1523521c70 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -349,11 +349,13 @@ EXPORT_SYMBOL(blk_put_queue);
 /**
  * blk_drain_queue - drain requests from request_queue
  * @q: queue to drain
+ * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
  *
- * Drain ELV_PRIV requests from @q.  The caller is responsible for ensuring
- * that no new requests which need to be drained are queued.
+ * Drain requests from @q.  If @drain_all is set, all requests are drained.
+ * If not, only ELVPRIV requests are drained.  The caller is responsible
+ * for ensuring that no new requests which need to be drained are queued.
  */
-void blk_drain_queue(struct request_queue *q)
+void blk_drain_queue(struct request_queue *q, bool drain_all)
 {
         while (true) {
                 int nr_rqs;
@@ -361,9 +363,15 @@ void blk_drain_queue(struct request_queue *q)
                 spin_lock_irq(q->queue_lock);
 
                 elv_drain_elevator(q);
+                if (drain_all)
+                        blk_throtl_drain(q);
 
                 __blk_run_queue(q);
-                nr_rqs = q->rq.elvpriv;
+
+                if (drain_all)
+                        nr_rqs = q->rq.count[0] + q->rq.count[1];
+                else
+                        nr_rqs = q->rq.elvpriv;
 
                 spin_unlock_irq(q->queue_lock);
 
@@ -373,30 +381,40 @@ void blk_drain_queue(struct request_queue *q)
         }
 }
 
-/*
- * Note: If a driver supplied the queue lock, it is disconnected
- * by this function. The actual state of the lock doesn't matter
- * here as the request_queue isn't accessible after this point
- * (QUEUE_FLAG_DEAD is set) and no other requests will be queued.
+/**
+ * blk_cleanup_queue - shutdown a request queue
+ * @q: request queue to shutdown
+ *
+ * Mark @q DEAD, drain all pending requests, destroy and put it.  All
+ * future requests will be failed immediately with -ENODEV.
  */
 void blk_cleanup_queue(struct request_queue *q)
 {
-        /*
-         * We know we have process context here, so we can be a little
-         * cautious and ensure that pending block actions on this device
-         * are done before moving on. Going into this function, we should
-         * not have processes doing IO to this device.
-         */
-        blk_sync_queue(q);
+        spinlock_t *lock = q->queue_lock;
 
-        del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
+        /* mark @q DEAD, no new request or merges will be allowed afterwards */
         mutex_lock(&q->sysfs_lock);
         queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
-        mutex_unlock(&q->sysfs_lock);
+
+        spin_lock_irq(lock);
+        queue_flag_set(QUEUE_FLAG_NOMERGES, q);
+        queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
+        queue_flag_set(QUEUE_FLAG_DEAD, q);
 
         if (q->queue_lock != &q->__queue_lock)
                 q->queue_lock = &q->__queue_lock;
 
+        spin_unlock_irq(lock);
+        mutex_unlock(&q->sysfs_lock);
+
+        /* drain all requests queued before DEAD marking */
+        blk_drain_queue(q, true);
+
+        /* @q won't process any more request, flush async actions */
+        del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
+        blk_sync_queue(q);
+
+        /* @q is and will stay empty, shutdown and put */
         blk_put_queue(q);
 }
 EXPORT_SYMBOL(blk_cleanup_queue);
@@ -1509,9 +1527,6 @@ generic_make_request_checks(struct bio *bio)
                 goto end_io;
         }
 
-        if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
-                goto end_io;
-
         part = bio->bi_bdev->bd_part;
         if (should_fail_request(part, bio->bi_size) ||
             should_fail_request(&part_to_disk(part)->part0,
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index a8eff5f8b9c5..e7f9f657f105 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -490,6 +490,7 @@ static void blk_release_queue(struct kobject *kobj)
         if (q->queue_tags)
                 __blk_queue_free_tags(q);
 
+        blk_throtl_release(q);
         blk_trace_shutdown(q);
 
         bdi_destroy(&q->backing_dev_info);
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 900a0c98745b..8edb9499b509 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -309,6 +309,10 @@ static struct throtl_grp * throtl_get_tg(struct throtl_data *td)
         struct blkio_cgroup *blkcg;
         struct request_queue *q = td->queue;
 
+        /* no throttling for dead queue */
+        if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
+                return NULL;
+
         rcu_read_lock();
         blkcg = task_blkio_cgroup(current);
         tg = throtl_find_tg(td, blkcg);
@@ -1001,11 +1005,6 @@ static void throtl_release_tgs(struct throtl_data *td)
         }
 }
 
-static void throtl_td_free(struct throtl_data *td)
-{
-        kfree(td);
-}
-
 /*
  * Blk cgroup controller notification saying that blkio_group object is being
  * delinked as associated cgroup object is going away. That also means that
@@ -1204,6 +1203,41 @@ out:
         return throttled;
 }
 
+/**
+ * blk_throtl_drain - drain throttled bios
+ * @q: request_queue to drain throttled bios for
+ *
+ * Dispatch all currently throttled bios on @q through ->make_request_fn().
+ */
+void blk_throtl_drain(struct request_queue *q)
+        __releases(q->queue_lock) __acquires(q->queue_lock)
+{
+        struct throtl_data *td = q->td;
+        struct throtl_rb_root *st = &td->tg_service_tree;
+        struct throtl_grp *tg;
+        struct bio_list bl;
+        struct bio *bio;
+
+        lockdep_is_held(q->queue_lock);
+
+        bio_list_init(&bl);
+
+        while ((tg = throtl_rb_first(st))) {
+                throtl_dequeue_tg(td, tg);
+
+                while ((bio = bio_list_peek(&tg->bio_lists[READ])))
+                        tg_dispatch_one_bio(td, tg, bio_data_dir(bio), &bl);
+                while ((bio = bio_list_peek(&tg->bio_lists[WRITE])))
+                        tg_dispatch_one_bio(td, tg, bio_data_dir(bio), &bl);
+        }
+        spin_unlock_irq(q->queue_lock);
+
+        while ((bio = bio_list_pop(&bl)))
+                generic_make_request(bio);
+
+        spin_lock_irq(q->queue_lock);
+}
+
 int blk_throtl_init(struct request_queue *q)
 {
         struct throtl_data *td;
@@ -1276,7 +1310,11 @@ void blk_throtl_exit(struct request_queue *q)
          * it.
          */
         throtl_shutdown_wq(q);
-        throtl_td_free(td);
+}
+
+void blk_throtl_release(struct request_queue *q)
+{
+        kfree(q->td);
 }
 
 static int __init throtl_init(void)
diff --git a/block/blk.h b/block/blk.h
index c018dba4e335..3f6551b3c92d 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -15,7 +15,7 @@ void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
                         struct bio *bio);
 int blk_rq_append_bio(struct request_queue *q, struct request *rq,
                       struct bio *bio);
-void blk_drain_queue(struct request_queue *q);
+void blk_drain_queue(struct request_queue *q, bool drain_all);
 void blk_dequeue_request(struct request *rq);
 void __blk_queue_free_tags(struct request_queue *q);
 bool __blk_end_bidi_request(struct request *rq, int error,
@@ -191,15 +191,19 @@ static inline int blk_do_io_stat(struct request *rq)
 
 #ifdef CONFIG_BLK_DEV_THROTTLING
 extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio);
+extern void blk_throtl_drain(struct request_queue *q);
 extern int blk_throtl_init(struct request_queue *q);
 extern void blk_throtl_exit(struct request_queue *q);
+extern void blk_throtl_release(struct request_queue *q);
 #else /* CONFIG_BLK_DEV_THROTTLING */
 static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
 {
         return false;
 }
+static inline void blk_throtl_drain(struct request_queue *q) { }
 static inline int blk_throtl_init(struct request_queue *q) { return 0; }
 static inline void blk_throtl_exit(struct request_queue *q) { }
+static inline void blk_throtl_release(struct request_queue *q) { }
 #endif /* CONFIG_BLK_DEV_THROTTLING */
 
 #endif /* BLK_INTERNAL_H */
diff --git a/block/elevator.c b/block/elevator.c
index 74a277ffed39..66343d6917d0 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -626,7 +626,7 @@ void elv_quiesce_start(struct request_queue *q)
         queue_flag_set(QUEUE_FLAG_ELVSWITCH, q);
         spin_unlock_irq(q->queue_lock);
 
-        blk_drain_queue(q);
+        blk_drain_queue(q, false);
 }
 
 void elv_quiesce_end(struct request_queue *q)