11 files changed, 75 insertions, 111 deletions

diff --git a/block/bio.c b/block/bio.c
index 8c2e55e39a1b..0ec61c9e536c 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -746,6 +746,14 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
 
                         goto done;
                 }
+
+                /*
+                 * If the queue doesn't support SG gaps and adding this
+                 * offset would create a gap, disallow it.
+                 */
+                if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) &&
+                    bvec_gap_to_prev(prev, offset))
+                        return 0;
         }
 
         if (bio->bi_vcnt >= bio->bi_max_vecs)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 069bc202ffe3..b9f4cc494ece 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -80,7 +80,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
         blkg->q = q;
         INIT_LIST_HEAD(&blkg->q_node);
         blkg->blkcg = blkcg;
-        blkg->refcnt = 1;
+        atomic_set(&blkg->refcnt, 1);
 
         /* root blkg uses @q->root_rl, init rl only for !root blkgs */
         if (blkcg != &blkcg_root) {
@@ -399,11 +399,8 @@ void __blkg_release_rcu(struct rcu_head *rcu_head)
 
         /* release the blkcg and parent blkg refs this blkg has been holding */
         css_put(&blkg->blkcg->css);
-        if (blkg->parent) {
-                spin_lock_irq(blkg->q->queue_lock);
+        if (blkg->parent)
                 blkg_put(blkg->parent);
-                spin_unlock_irq(blkg->q->queue_lock);
-        }
 
         blkg_free(blkg);
 }
@@ -1093,7 +1090,7 @@ EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
  * Register @pol with blkcg core.  Might sleep and @pol may be modified on
  * successful registration.  Returns 0 on success and -errno on failure.
  */
-int __init blkcg_policy_register(struct blkcg_policy *pol)
+int blkcg_policy_register(struct blkcg_policy *pol)
 {
         int i, ret;
 
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index cbb7f943f78a..d3fd7aa3d2a3 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -18,6 +18,7 @@
 #include <linux/seq_file.h>
 #include <linux/radix-tree.h>
 #include <linux/blkdev.h>
+#include <linux/atomic.h>
 
 /* Max limits for throttle policy */
 #define THROTL_IOPS_MAX         UINT_MAX
@@ -104,7 +105,7 @@ struct blkcg_gq {
         struct request_list             rl;
 
         /* reference count */
-        int                             refcnt;
+        atomic_t                        refcnt;
 
         /* is this blkg online? protected by both blkcg and q locks */
         bool                            online;
@@ -145,7 +146,7 @@ void blkcg_drain_queue(struct request_queue *q);
 void blkcg_exit_queue(struct request_queue *q);
 
 /* Blkio controller policy registration */
-int __init blkcg_policy_register(struct blkcg_policy *pol);
+int blkcg_policy_register(struct blkcg_policy *pol);
 void blkcg_policy_unregister(struct blkcg_policy *pol);
 int blkcg_activate_policy(struct request_queue *q,
                           const struct blkcg_policy *pol);
@@ -257,13 +258,12 @@ static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
  * blkg_get - get a blkg reference
  * @blkg: blkg to get
  *
- * The caller should be holding queue_lock and an existing reference.
+ * The caller should be holding an existing reference.
  */
 static inline void blkg_get(struct blkcg_gq *blkg)
 {
-        lockdep_assert_held(blkg->q->queue_lock);
-        WARN_ON_ONCE(!blkg->refcnt);
-        blkg->refcnt++;
+        WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
+        atomic_inc(&blkg->refcnt);
 }
 
 void __blkg_release_rcu(struct rcu_head *rcu);
@@ -271,14 +271,11 @@ void __blkg_release_rcu(struct rcu_head *rcu);
 /**
  * blkg_put - put a blkg reference
  * @blkg: blkg to put
- *
- * The caller should be holding queue_lock.
  */
 static inline void blkg_put(struct blkcg_gq *blkg)
 {
-        lockdep_assert_held(blkg->q->queue_lock);
-        WARN_ON_ONCE(blkg->refcnt <= 0);
-        if (!--blkg->refcnt)
+        WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0);
+        if (atomic_dec_and_test(&blkg->refcnt))
                 call_rcu(&blkg->rcu_head, __blkg_release_rcu);
 }
 
@@ -580,7 +577,7 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { ret
 static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
 static inline void blkcg_drain_queue(struct request_queue *q) { }
 static inline void blkcg_exit_queue(struct request_queue *q) { }
-static inline int __init blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
+static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
 static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
 static inline int blkcg_activate_policy(struct request_queue *q,
                                         const struct blkcg_policy *pol) { return 0; }
diff --git a/block/blk-core.c b/block/blk-core.c
index f6f6b9af3e3f..6f8dba161bfe 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -3312,8 +3312,7 @@ int __init blk_dev_init(void)
 
         /* used for unplugging and affects IO latency/throughput - HIGHPRI */
         kblockd_workqueue = alloc_workqueue("kblockd",
-                                            WQ_MEM_RECLAIM | WQ_HIGHPRI |
-                                            WQ_POWER_EFFICIENT, 0);
+                                            WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
         if (!kblockd_workqueue)
                 panic("Failed to create kblockd\n");
 
diff --git a/block/blk-flush.c b/block/blk-flush.c
index 8ffee4b5f93d..3cb5e9e7108a 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -422,44 +422,6 @@ void blk_insert_flush(struct request *rq)
 }
 
 /**
- * blk_abort_flushes - @q is being aborted, abort flush requests
- * @q: request_queue being aborted
- *
- * To be called from elv_abort_queue().  @q is being aborted.  Prepare all
- * FLUSH/FUA requests for abortion.
- *
- * CONTEXT:
- * spin_lock_irq(q->queue_lock)
- */
-void blk_abort_flushes(struct request_queue *q)
-{
-        struct request *rq, *n;
-        int i;
-
-        /*
-         * Requests in flight for data are already owned by the dispatch
-         * queue or the device driver.  Just restore for normal completion.
-         */
-        list_for_each_entry_safe(rq, n, &q->flush_data_in_flight, flush.list) {
-                list_del_init(&rq->flush.list);
-                blk_flush_restore_request(rq);
-        }
-
-        /*
-         * We need to give away requests on flush queues.  Restore for
-         * normal completion and put them on the dispatch queue.
-         */
-        for (i = 0; i < ARRAY_SIZE(q->flush_queue); i++) {
-                list_for_each_entry_safe(rq, n, &q->flush_queue[i],
-                                         flush.list) {
-                        list_del_init(&rq->flush.list);
-                        blk_flush_restore_request(rq);
-                        list_add_tail(&rq->queuelist, &q->queue_head);
-                }
-        }
-}
-
-/**
  * blkdev_issue_flush - queue a flush
  * @bdev:       blockdev to issue flush for
  * @gfp_mask:   memory allocation flags (for bio_alloc)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index b3bf0df0f4c2..54535831f1e1 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -568,6 +568,8 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
 
 bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
 {
+        struct request_queue *q = rq->q;
+
         if (!rq_mergeable(rq) || !bio_mergeable(bio))
                 return false;
 
@@ -591,6 +593,14 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
             !blk_write_same_mergeable(rq->bio, bio))
                 return false;
 
+        if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS)) {
+                struct bio_vec *bprev;
+
+                bprev = &rq->biotail->bi_io_vec[bio->bi_vcnt - 1];
+                if (bvec_gap_to_prev(bprev, bio->bi_io_vec[0].bv_offset))
+                        return false;
+        }
+
         return true;
 }
 
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 1aab39f71d95..c1b92426c95e 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -43,9 +43,16 @@ bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
         return bt_has_free_tags(&tags->bitmap_tags);
 }
 
-static inline void bt_index_inc(unsigned int *index)
+static inline int bt_index_inc(int index)
 {
-        *index = (*index + 1) & (BT_WAIT_QUEUES - 1);
+        return (index + 1) & (BT_WAIT_QUEUES - 1);
+}
+
+static inline void bt_index_atomic_inc(atomic_t *index)
+{
+        int old = atomic_read(index);
+        int new = bt_index_inc(old);
+        atomic_cmpxchg(index, old, new);
 }
 
 /*
@@ -69,14 +76,14 @@ static void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags)
         int i, wake_index;
 
         bt = &tags->bitmap_tags;
-        wake_index = bt->wake_index;
+        wake_index = atomic_read(&bt->wake_index);
         for (i = 0; i < BT_WAIT_QUEUES; i++) {
                 struct bt_wait_state *bs = &bt->bs[wake_index];
 
                 if (waitqueue_active(&bs->wait))
                         wake_up(&bs->wait);
 
-                bt_index_inc(&wake_index);
+                wake_index = bt_index_inc(wake_index);
         }
 }
 
@@ -212,12 +219,14 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt,
                                          struct blk_mq_hw_ctx *hctx)
 {
         struct bt_wait_state *bs;
+        int wait_index;
 
         if (!hctx)
                 return &bt->bs[0];
 
-        bs = &bt->bs[hctx->wait_index];
-        bt_index_inc(&hctx->wait_index);
+        wait_index = atomic_read(&hctx->wait_index);
+        bs = &bt->bs[wait_index];
+        bt_index_atomic_inc(&hctx->wait_index);
         return bs;
 }
 
@@ -239,18 +248,12 @@ static int bt_get(struct blk_mq_alloc_data *data,
 
         bs = bt_wait_ptr(bt, hctx);
         do {
-                bool was_empty;
-
-                was_empty = list_empty(&wait.task_list);
                 prepare_to_wait(&bs->wait, &wait, TASK_UNINTERRUPTIBLE);
 
                 tag = __bt_get(hctx, bt, last_tag);
                 if (tag != -1)
                         break;
 
-                if (was_empty)
-                        atomic_set(&bs->wait_cnt, bt->wake_cnt);
-
                 blk_mq_put_ctx(data->ctx);
 
                 io_schedule();
@@ -313,18 +316,19 @@ static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt)
 {
         int i, wake_index;
 
-        wake_index = bt->wake_index;
+        wake_index = atomic_read(&bt->wake_index);
         for (i = 0; i < BT_WAIT_QUEUES; i++) {
                 struct bt_wait_state *bs = &bt->bs[wake_index];
 
                 if (waitqueue_active(&bs->wait)) {
-                        if (wake_index != bt->wake_index)
-                                bt->wake_index = wake_index;
+                        int o = atomic_read(&bt->wake_index);
+                        if (wake_index != o)
+                                atomic_cmpxchg(&bt->wake_index, o, wake_index);
 
                         return bs;
                 }
 
-                bt_index_inc(&wake_index);
+                wake_index = bt_index_inc(wake_index);
         }
 
         return NULL;
@@ -334,6 +338,7 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag)
 {
         const int index = TAG_TO_INDEX(bt, tag);
         struct bt_wait_state *bs;
+        int wait_cnt;
 
         /*
          * The unlock memory barrier need to order access to req in free
@@ -342,10 +347,19 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag)
         clear_bit_unlock(TAG_TO_BIT(bt, tag), &bt->map[index].word);
 
         bs = bt_wake_ptr(bt);
-        if (bs && atomic_dec_and_test(&bs->wait_cnt)) {
-                atomic_set(&bs->wait_cnt, bt->wake_cnt);
-                bt_index_inc(&bt->wake_index);
+        if (!bs)
+                return;
+
+        wait_cnt = atomic_dec_return(&bs->wait_cnt);
+        if (wait_cnt == 0) {
+wake:
+                atomic_add(bt->wake_cnt, &bs->wait_cnt);
+                bt_index_atomic_inc(&bt->wake_index);
                 wake_up(&bs->wait);
+        } else if (wait_cnt < 0) {
+                wait_cnt = atomic_inc_return(&bs->wait_cnt);
+                if (!wait_cnt)
+                        goto wake;
         }
 }
 
@@ -499,10 +513,13 @@ static int bt_alloc(struct blk_mq_bitmap_tags *bt, unsigned int depth,
                 return -ENOMEM;
         }
 
-        for (i = 0; i < BT_WAIT_QUEUES; i++)
+        bt_update_count(bt, depth);
+
+        for (i = 0; i < BT_WAIT_QUEUES; i++) {
                 init_waitqueue_head(&bt->bs[i].wait);
+                atomic_set(&bt->bs[i].wait_cnt, bt->wake_cnt);
+        }
 
-        bt_update_count(bt, depth);
         return 0;
 }
 
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 98696a65d4d4..6206ed17ef76 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -24,7 +24,7 @@ struct blk_mq_bitmap_tags {
         unsigned int map_nr;
         struct blk_align_bitmap *map;
 
-        unsigned int wake_index;
+        atomic_t wake_index;
         struct bt_wait_state *bs;
 };
 
diff --git a/block/blk-mq.c b/block/blk-mq.c
index e11f5f8e0313..ad69ef657e85 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -109,7 +109,7 @@ static void blk_mq_queue_exit(struct request_queue *q)
         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 }
 
-static void __blk_mq_drain_queue(struct request_queue *q)
+void blk_mq_drain_queue(struct request_queue *q)
 {
         while (true) {
                 s64 count;
@@ -120,7 +120,7 @@ static void __blk_mq_drain_queue(struct request_queue *q)
 
                 if (count == 0)
                         break;
-                blk_mq_run_queues(q, false);
+                blk_mq_start_hw_queues(q);
                 msleep(10);
         }
 }
@@ -139,12 +139,7 @@ static void blk_mq_freeze_queue(struct request_queue *q)
         spin_unlock_irq(q->queue_lock);
 
         if (drain)
-                __blk_mq_drain_queue(q);
-}
-
-void blk_mq_drain_queue(struct request_queue *q)
-{
-        __blk_mq_drain_queue(q);
+                blk_mq_drain_queue(q);
 }
 
 static void blk_mq_unfreeze_queue(struct request_queue *q)
@@ -883,7 +878,7 @@ void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 
         preempt_disable();
-        __blk_mq_run_hw_queue(hctx);
+        blk_mq_run_hw_queue(hctx, false);
         preempt_enable();
 }
 EXPORT_SYMBOL(blk_mq_start_hw_queue);
diff --git a/block/blk.h b/block/blk.h
index 45385e9abf6f..6748c4f8d7a1 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -84,7 +84,6 @@ static inline void blk_clear_rq_complete(struct request *rq)
 #define ELV_ON_HASH(rq) ((rq)->cmd_flags & REQ_HASHED)
 
 void blk_insert_flush(struct request *rq);
-void blk_abort_flushes(struct request_queue *q);
 
 static inline struct request *__elv_next_request(struct request_queue *q)
 {
diff --git a/block/elevator.c b/block/elevator.c
index f35edddfe9b5..24c28b659bb3 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -729,26 +729,6 @@ int elv_may_queue(struct request_queue *q, int rw)
         return ELV_MQUEUE_MAY;
 }
 
-void elv_abort_queue(struct request_queue *q)
-{
-        struct request *rq;
-
-        blk_abort_flushes(q);
-
-        while (!list_empty(&q->queue_head)) {
-                rq = list_entry_rq(q->queue_head.next);
-                rq->cmd_flags |= REQ_QUIET;
-                trace_block_rq_abort(q, rq);
-                /*
-                 * Mark this request as started so we don't trigger
-                 * any debug logic in the end I/O path.
-                 */
-                blk_start_request(rq);
-                __blk_end_request_all(rq, -EIO);
-        }
-}
-EXPORT_SYMBOL(elv_abort_queue);
-
 void elv_completed_request(struct request_queue *q, struct request *rq)
 {
         struct elevator_queue *e = q->elevator;
@@ -845,7 +825,7 @@ void elv_unregister_queue(struct request_queue *q)
 }
 EXPORT_SYMBOL(elv_unregister_queue);
 
-int __init elv_register(struct elevator_type *e)
+int elv_register(struct elevator_type *e)
 {
         char *def = "";