Merge tag 'for-5.3/block-20190708' of git://git.kernel.dk/linux-block

Pull block updates from Jens Axboe:
 "This is the main block updates for 5.3. Nothing earth shattering or
  major in here, just fixes, additions, and improvements all over the
  map. This contains:

   - Series of documentation fixes (Bart)

   - Optimization of the blk-mq ctx get/put (Bart)

   - null_blk removal race condition fix (Bob)

   - req/bio_op() cleanups (Chaitanya)

   - Series cleaning up the segment accounting, and request/bio mapping
     (Christoph)

   - Series cleaning up the page getting/putting for bios (Christoph)

   - block cgroup cleanups and moving it to where it is used (Christoph)

   - block cgroup fixes (Tejun)

   - Series of fixes and improvements to bcache, most notably a write
     deadlock fix (Coly)

   - blk-iolatency STS_AGAIN and accounting fixes (Dennis)

   - Series of improvements and fixes to BFQ (Douglas, Paolo)

   - debugfs_create() return value check removal for drbd (Greg)

   - Use struct_size(), where appropriate (Gustavo)

   - Two lighnvm fixes (Heiner, Geert)

   - MD fixes, including a read balance and corruption fix (Guoqing,
     Marcos, Xiao, Yufen)

   - block opal shadow mbr additions (Jonas, Revanth)

   - sbitmap compare-and-exhange improvemnts (Pavel)

   - Fix for potential bio->bi_size overflow (Ming)

   - NVMe pull requests:
       - improved PCIe suspent support (Keith Busch)
       - error injection support for the admin queue (Akinobu Mita)
       - Fibre Channel discovery improvements (James Smart)
       - tracing improvements including nvmetc tracing support (Minwoo Im)
       - misc fixes and cleanups (Anton Eidelman, Minwoo Im, Chaitanya
         Kulkarni)"

   - Various little fixes and improvements to drivers and core"

* tag 'for-5.3/block-20190708' of git://git.kernel.dk/linux-block: (153 commits)
  blk-iolatency: fix STS_AGAIN handling
  block: nr_phys_segments needs to be zero for REQ_OP_WRITE_ZEROES
  blk-mq: simplify blk_mq_make_request()
  blk-mq: remove blk_mq_put_ctx()
  sbitmap: Replace cmpxchg with xchg
  block: fix .bi_size overflow
  block: sed-opal: check size of shadow mbr
  block: sed-opal: ioctl for writing to shadow mbr
  block: sed-opal: add ioctl for done-mark of shadow mbr
  block: never take page references for ITER_BVEC
  direct-io: use bio_release_pages in dio_bio_complete
  block_dev: use bio_release_pages in bio_unmap_user
  block_dev: use bio_release_pages in blkdev_bio_end_io
  iomap: use bio_release_pages in iomap_dio_bio_end_io
  block: use bio_release_pages in bio_map_user_iov
  block: use bio_release_pages in bio_unmap_user
  block: optionally mark pages dirty in bio_release_pages
  block: move the BIO_NO_PAGE_REF check into bio_release_pages
  block: skd_main.c: Remove call to memset after dma_alloc_coherent
  block: mtip32xx: Remove call to memset after dma_alloc_coherent
  ...

22 files changed, 1342 insertions, 818 deletions

diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched
index 4626b88b2d5a..7a6b2f29a582 100644
--- a/block/Kconfig.iosched
+++ b/block/Kconfig.iosched
@@ -36,6 +36,13 @@ config BFQ_GROUP_IOSCHED
        Enable hierarchical scheduling in BFQ, using the blkio
        (cgroups-v1) or io (cgroups-v2) controller.
 
+config BFQ_CGROUP_DEBUG
+        bool "BFQ IO controller debugging"
+        depends on BFQ_GROUP_IOSCHED
+        ---help---
+        Enable some debugging help. Currently it exports additional stat
+        files in a cgroup which can be useful for debugging.
+
 endmenu
 
 endif
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index b3796a40a61a..0f6cd688924f 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -15,7 +15,83 @@
 
 #include "bfq-iosched.h"
 
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) &&  defined(CONFIG_DEBUG_BLK_CGROUP)
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
+static int bfq_stat_init(struct bfq_stat *stat, gfp_t gfp)
+{
+        int ret;
+
+        ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
+        if (ret)
+                return ret;
+
+        atomic64_set(&stat->aux_cnt, 0);
+        return 0;
+}
+
+static void bfq_stat_exit(struct bfq_stat *stat)
+{
+        percpu_counter_destroy(&stat->cpu_cnt);
+}
+
+/**
+ * bfq_stat_add - add a value to a bfq_stat
+ * @stat: target bfq_stat
+ * @val: value to add
+ *
+ * Add @val to @stat.  The caller must ensure that IRQ on the same CPU
+ * don't re-enter this function for the same counter.
+ */
+static inline void bfq_stat_add(struct bfq_stat *stat, uint64_t val)
+{
+        percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
+}
+
+/**
+ * bfq_stat_read - read the current value of a bfq_stat
+ * @stat: bfq_stat to read
+ */
+static inline uint64_t bfq_stat_read(struct bfq_stat *stat)
+{
+        return percpu_counter_sum_positive(&stat->cpu_cnt);
+}
+
+/**
+ * bfq_stat_reset - reset a bfq_stat
+ * @stat: bfq_stat to reset
+ */
+static inline void bfq_stat_reset(struct bfq_stat *stat)
+{
+        percpu_counter_set(&stat->cpu_cnt, 0);
+        atomic64_set(&stat->aux_cnt, 0);
+}
+
+/**
+ * bfq_stat_add_aux - add a bfq_stat into another's aux count
+ * @to: the destination bfq_stat
+ * @from: the source
+ *
+ * Add @from's count including the aux one to @to's aux count.
+ */
+static inline void bfq_stat_add_aux(struct bfq_stat *to,
+                                     struct bfq_stat *from)
+{
+        atomic64_add(bfq_stat_read(from) + atomic64_read(&from->aux_cnt),
+                     &to->aux_cnt);
+}
+
+/**
+ * blkg_prfill_stat - prfill callback for bfq_stat
+ * @sf: seq_file to print to
+ * @pd: policy private data of interest
+ * @off: offset to the bfq_stat in @pd
+ *
+ * prfill callback for printing a bfq_stat.
+ */
+static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
+                int off)
+{
+        return __blkg_prfill_u64(sf, pd, bfq_stat_read((void *)pd + off));
+}
 
 /* bfqg stats flags */
 enum bfqg_stats_flags {
@@ -53,7 +129,7 @@ static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
 
         now = ktime_get_ns();
         if (now > stats->start_group_wait_time)
-                blkg_stat_add(&stats->group_wait_time,
+                bfq_stat_add(&stats->group_wait_time,
                               now - stats->start_group_wait_time);
         bfqg_stats_clear_waiting(stats);
 }
@@ -82,14 +158,14 @@ static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
 
         now = ktime_get_ns();
         if (now > stats->start_empty_time)
-                blkg_stat_add(&stats->empty_time,
+                bfq_stat_add(&stats->empty_time,
                               now - stats->start_empty_time);
         bfqg_stats_clear_empty(stats);
 }
 
 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
 {
-        blkg_stat_add(&bfqg->stats.dequeue, 1);
+        bfq_stat_add(&bfqg->stats.dequeue, 1);
 }
 
 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
@@ -119,7 +195,7 @@ void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
                 u64 now = ktime_get_ns();
 
                 if (now > stats->start_idle_time)
-                        blkg_stat_add(&stats->idle_time,
+                        bfq_stat_add(&stats->idle_time,
                                       now - stats->start_idle_time);
                 bfqg_stats_clear_idling(stats);
         }
@@ -137,9 +213,9 @@ void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)
 {
         struct bfqg_stats *stats = &bfqg->stats;
 
-        blkg_stat_add(&stats->avg_queue_size_sum,
+        bfq_stat_add(&stats->avg_queue_size_sum,
                       blkg_rwstat_total(&stats->queued));
-        blkg_stat_add(&stats->avg_queue_size_samples, 1);
+        bfq_stat_add(&stats->avg_queue_size_samples, 1);
         bfqg_stats_update_group_wait_time(stats);
 }
 
@@ -176,7 +252,7 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
                                 io_start_time_ns - start_time_ns);
 }
 
-#else /* CONFIG_BFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
+#else /* CONFIG_BFQ_CGROUP_DEBUG */
 
 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
                               unsigned int op) { }
@@ -190,7 +266,7 @@ void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }
 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }
 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }
 
-#endif /* CONFIG_BFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
+#endif /* CONFIG_BFQ_CGROUP_DEBUG */
 
 #ifdef CONFIG_BFQ_GROUP_IOSCHED
 
@@ -274,18 +350,18 @@ void bfqg_and_blkg_put(struct bfq_group *bfqg)
 /* @stats = 0 */
 static void bfqg_stats_reset(struct bfqg_stats *stats)
 {
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         /* queued stats shouldn't be cleared */
         blkg_rwstat_reset(&stats->merged);
         blkg_rwstat_reset(&stats->service_time);
         blkg_rwstat_reset(&stats->wait_time);
-        blkg_stat_reset(&stats->time);
-        blkg_stat_reset(&stats->avg_queue_size_sum);
-        blkg_stat_reset(&stats->avg_queue_size_samples);
-        blkg_stat_reset(&stats->dequeue);
-        blkg_stat_reset(&stats->group_wait_time);
-        blkg_stat_reset(&stats->idle_time);
-        blkg_stat_reset(&stats->empty_time);
+        bfq_stat_reset(&stats->time);
+        bfq_stat_reset(&stats->avg_queue_size_sum);
+        bfq_stat_reset(&stats->avg_queue_size_samples);
+        bfq_stat_reset(&stats->dequeue);
+        bfq_stat_reset(&stats->group_wait_time);
+        bfq_stat_reset(&stats->idle_time);
+        bfq_stat_reset(&stats->empty_time);
 #endif
 }
 
@@ -295,19 +371,19 @@ static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from)
         if (!to || !from)
                 return;
 
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         /* queued stats shouldn't be cleared */
         blkg_rwstat_add_aux(&to->merged, &from->merged);
         blkg_rwstat_add_aux(&to->service_time, &from->service_time);
         blkg_rwstat_add_aux(&to->wait_time, &from->wait_time);
-        blkg_stat_add_aux(&from->time, &from->time);
-        blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
-        blkg_stat_add_aux(&to->avg_queue_size_samples,
+        bfq_stat_add_aux(&from->time, &from->time);
+        bfq_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
+        bfq_stat_add_aux(&to->avg_queue_size_samples,
                           &from->avg_queue_size_samples);
-        blkg_stat_add_aux(&to->dequeue, &from->dequeue);
-        blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
-        blkg_stat_add_aux(&to->idle_time, &from->idle_time);
-        blkg_stat_add_aux(&to->empty_time, &from->empty_time);
+        bfq_stat_add_aux(&to->dequeue, &from->dequeue);
+        bfq_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
+        bfq_stat_add_aux(&to->idle_time, &from->idle_time);
+        bfq_stat_add_aux(&to->empty_time, &from->empty_time);
 #endif
 }
 
@@ -355,35 +431,35 @@ void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
 
 static void bfqg_stats_exit(struct bfqg_stats *stats)
 {
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         blkg_rwstat_exit(&stats->merged);
         blkg_rwstat_exit(&stats->service_time);
         blkg_rwstat_exit(&stats->wait_time);
         blkg_rwstat_exit(&stats->queued);
-        blkg_stat_exit(&stats->time);
-        blkg_stat_exit(&stats->avg_queue_size_sum);
-        blkg_stat_exit(&stats->avg_queue_size_samples);
-        blkg_stat_exit(&stats->dequeue);
-        blkg_stat_exit(&stats->group_wait_time);
-        blkg_stat_exit(&stats->idle_time);
-        blkg_stat_exit(&stats->empty_time);
+        bfq_stat_exit(&stats->time);
+        bfq_stat_exit(&stats->avg_queue_size_sum);
+        bfq_stat_exit(&stats->avg_queue_size_samples);
+        bfq_stat_exit(&stats->dequeue);
+        bfq_stat_exit(&stats->group_wait_time);
+        bfq_stat_exit(&stats->idle_time);
+        bfq_stat_exit(&stats->empty_time);
 #endif
 }
 
 static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
 {
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         if (blkg_rwstat_init(&stats->merged, gfp) ||
             blkg_rwstat_init(&stats->service_time, gfp) ||
             blkg_rwstat_init(&stats->wait_time, gfp) ||
             blkg_rwstat_init(&stats->queued, gfp) ||
-            blkg_stat_init(&stats->time, gfp) ||
-            blkg_stat_init(&stats->avg_queue_size_sum, gfp) ||
-            blkg_stat_init(&stats->avg_queue_size_samples, gfp) ||
-            blkg_stat_init(&stats->dequeue, gfp) ||
-            blkg_stat_init(&stats->group_wait_time, gfp) ||
-            blkg_stat_init(&stats->idle_time, gfp) ||
-            blkg_stat_init(&stats->empty_time, gfp)) {
+            bfq_stat_init(&stats->time, gfp) ||
+            bfq_stat_init(&stats->avg_queue_size_sum, gfp) ||
+            bfq_stat_init(&stats->avg_queue_size_samples, gfp) ||
+            bfq_stat_init(&stats->dequeue, gfp) ||
+            bfq_stat_init(&stats->group_wait_time, gfp) ||
+            bfq_stat_init(&stats->idle_time, gfp) ||
+            bfq_stat_init(&stats->empty_time, gfp)) {
                 bfqg_stats_exit(stats);
                 return -ENOMEM;
         }
@@ -909,7 +985,7 @@ static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,
         return ret ?: nbytes;
 }
 
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
 static int bfqg_print_stat(struct seq_file *sf, void *v)
 {
         blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
@@ -927,17 +1003,34 @@ static int bfqg_print_rwstat(struct seq_file *sf, void *v)
 static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,
                                       struct blkg_policy_data *pd, int off)
 {
-        u64 sum = blkg_stat_recursive_sum(pd_to_blkg(pd),
-                                          &blkcg_policy_bfq, off);
+        struct blkcg_gq *blkg = pd_to_blkg(pd);
+        struct blkcg_gq *pos_blkg;
+        struct cgroup_subsys_state *pos_css;
+        u64 sum = 0;
+
+        lockdep_assert_held(&blkg->q->queue_lock);
+
+        rcu_read_lock();
+        blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
+                struct bfq_stat *stat;
+
+                if (!pos_blkg->online)
+                        continue;
+
+                stat = (void *)blkg_to_pd(pos_blkg, &blkcg_policy_bfq) + off;
+                sum += bfq_stat_read(stat) + atomic64_read(&stat->aux_cnt);
+        }
+        rcu_read_unlock();
+
         return __blkg_prfill_u64(sf, pd, sum);
 }
 
 static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,
                                         struct blkg_policy_data *pd, int off)
 {
-        struct blkg_rwstat sum = blkg_rwstat_recursive_sum(pd_to_blkg(pd),
-                                                           &blkcg_policy_bfq,
-                                                           off);
+        struct blkg_rwstat_sample sum;
+
+        blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);
         return __blkg_prfill_rwstat(sf, pd, &sum);
 }
 
@@ -975,12 +1068,13 @@ static int bfqg_print_stat_sectors(struct seq_file *sf, void *v)
 static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf,
                                          struct blkg_policy_data *pd, int off)
 {
-        struct blkg_rwstat tmp = blkg_rwstat_recursive_sum(pd->blkg, NULL,
-                                        offsetof(struct blkcg_gq, stat_bytes));
-        u64 sum = atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +
-                atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);
+        struct blkg_rwstat_sample tmp;
 
-        return __blkg_prfill_u64(sf, pd, sum >> 9);
+        blkg_rwstat_recursive_sum(pd->blkg, NULL,
+                        offsetof(struct blkcg_gq, stat_bytes), &tmp);
+
+        return __blkg_prfill_u64(sf, pd,
+                (tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);
 }
 
 static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v)
@@ -995,11 +1089,11 @@ static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf,
                                       struct blkg_policy_data *pd, int off)
 {
         struct bfq_group *bfqg = pd_to_bfqg(pd);
-        u64 samples = blkg_stat_read(&bfqg->stats.avg_queue_size_samples);
+        u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);
         u64 v = 0;
 
         if (samples) {
-                v = blkg_stat_read(&bfqg->stats.avg_queue_size_sum);
+                v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);
                 v = div64_u64(v, samples);
         }
         __blkg_prfill_u64(sf, pd, v);
@@ -1014,7 +1108,7 @@ static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v)
                           0, false);
         return 0;
 }
-#endif /* CONFIG_DEBUG_BLK_CGROUP */
+#endif /* CONFIG_BFQ_CGROUP_DEBUG */
 
 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
 {
@@ -1062,7 +1156,7 @@ struct cftype bfq_blkcg_legacy_files[] = {
                 .private = (unsigned long)&blkcg_policy_bfq,
                 .seq_show = blkg_print_stat_ios,
         },
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         {
                 .name = "bfq.time",
                 .private = offsetof(struct bfq_group, stats.time),
@@ -1092,7 +1186,7 @@ struct cftype bfq_blkcg_legacy_files[] = {
                 .private = offsetof(struct bfq_group, stats.queued),
                 .seq_show = bfqg_print_rwstat,
         },
-#endif /* CONFIG_DEBUG_BLK_CGROUP */
+#endif /* CONFIG_BFQ_CGROUP_DEBUG */
 
         /* the same statistics which cover the bfqg and its descendants */
         {
@@ -1105,7 +1199,7 @@ struct cftype bfq_blkcg_legacy_files[] = {
                 .private = (unsigned long)&blkcg_policy_bfq,
                 .seq_show = blkg_print_stat_ios_recursive,
         },
-#ifdef CONFIG_DEBUG_BLK_CGROUP
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         {
                 .name = "bfq.time_recursive",
                 .private = offsetof(struct bfq_group, stats.time),
@@ -1159,7 +1253,7 @@ struct cftype bfq_blkcg_legacy_files[] = {
                 .private = offsetof(struct bfq_group, stats.dequeue),
                 .seq_show = bfqg_print_stat,
         },
-#endif  /* CONFIG_DEBUG_BLK_CGROUP */
+#endif  /* CONFIG_BFQ_CGROUP_DEBUG */
         { }     /* terminate */
 };
 
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index f9269ae6da9c..50c9d2598500 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -157,6 +157,7 @@ BFQ_BFQQ_FNS(in_large_burst);
 BFQ_BFQQ_FNS(coop);
 BFQ_BFQQ_FNS(split_coop);
 BFQ_BFQQ_FNS(softrt_update);
+BFQ_BFQQ_FNS(has_waker);
 #undef BFQ_BFQQ_FNS                                             \
 
 /* Expiration time of sync (0) and async (1) requests, in ns. */
@@ -1427,17 +1428,19 @@ static int bfq_min_budget(struct bfq_data *bfqd)
  * mechanism may be re-designed in such a way to make it possible to
  * know whether preemption is needed without needing to update service
  * trees). In addition, queue preemptions almost always cause random
- * I/O, and thus loss of throughput. Because of these facts, the next
- * function adopts the following simple scheme to avoid both costly
- * operations and too frequent preemptions: it requests the expiration
- * of the in-service queue (unconditionally) only for queues that need
- * to recover a hole, or that either are weight-raised or deserve to
- * be weight-raised.
+ * I/O, which may in turn cause loss of throughput. Finally, there may
+ * even be no in-service queue when the next function is invoked (so,
+ * no queue to compare timestamps with). Because of these facts, the
+ * next function adopts the following simple scheme to avoid costly
+ * operations, too frequent preemptions and too many dependencies on
+ * the state of the scheduler: it requests the expiration of the
+ * in-service queue (unconditionally) only for queues that need to
+ * recover a hole. Then it delegates to other parts of the code the
+ * responsibility of handling the above case 2.
  */
 static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd,
                                                 struct bfq_queue *bfqq,
-                                                bool arrived_in_time,
-                                                bool wr_or_deserves_wr)
+                                                bool arrived_in_time)
 {
         struct bfq_entity *entity = &bfqq->entity;
 
@@ -1492,7 +1495,7 @@ static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd,
         entity->budget = max_t(unsigned long, bfqq->max_budget,
                                bfq_serv_to_charge(bfqq->next_rq, bfqq));
         bfq_clear_bfqq_non_blocking_wait_rq(bfqq);
-        return wr_or_deserves_wr;
+        return false;
 }
 
 /*
@@ -1610,6 +1613,36 @@ static bool bfq_bfqq_idle_for_long_time(struct bfq_data *bfqd,
                         bfqd->bfq_wr_min_idle_time);
 }
 
+
+/*
+ * Return true if bfqq is in a higher priority class, or has a higher
+ * weight than the in-service queue.
+ */
+static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq,
+                                            struct bfq_queue *in_serv_bfqq)
+{
+        int bfqq_weight, in_serv_weight;
+
+        if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class)
+                return true;
+
+        if (in_serv_bfqq->entity.parent == bfqq->entity.parent) {
+                bfqq_weight = bfqq->entity.weight;
+                in_serv_weight = in_serv_bfqq->entity.weight;
+        } else {
+                if (bfqq->entity.parent)
+                        bfqq_weight = bfqq->entity.parent->weight;
+                else
+                        bfqq_weight = bfqq->entity.weight;
+                if (in_serv_bfqq->entity.parent)
+                        in_serv_weight = in_serv_bfqq->entity.parent->weight;
+                else
+                        in_serv_weight = in_serv_bfqq->entity.weight;
+        }
+
+        return bfqq_weight > in_serv_weight;
+}
+
 static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
                                              struct bfq_queue *bfqq,
                                              int old_wr_coeff,
@@ -1654,8 +1687,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
          */
         bfqq_wants_to_preempt =
                 bfq_bfqq_update_budg_for_activation(bfqd, bfqq,
-                                                    arrived_in_time,
-                                                    wr_or_deserves_wr);
+                                                    arrived_in_time);
 
         /*
          * If bfqq happened to be activated in a burst, but has been
@@ -1720,21 +1752,111 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd,
 
         /*
          * Expire in-service queue only if preemption may be needed
-         * for guarantees. In this respect, the function
-         * next_queue_may_preempt just checks a simple, necessary
-         * condition, and not a sufficient condition based on
-         * timestamps. In fact, for the latter condition to be
-         * evaluated, timestamps would need first to be updated, and
-         * this operation is quite costly (see the comments on the
-         * function bfq_bfqq_update_budg_for_activation).
+         * for guarantees. In particular, we care only about two
+         * cases. The first is that bfqq has to recover a service
+         * hole, as explained in the comments on
+         * bfq_bfqq_update_budg_for_activation(), i.e., that
+         * bfqq_wants_to_preempt is true. However, if bfqq does not
+         * carry time-critical I/O, then bfqq's bandwidth is less
+         * important than that of queues that carry time-critical I/O.
+         * So, as a further constraint, we consider this case only if
+         * bfqq is at least as weight-raised, i.e., at least as time
+         * critical, as the in-service queue.
+         *
+         * The second case is that bfqq is in a higher priority class,
+         * or has a higher weight than the in-service queue. If this
+         * condition does not hold, we don't care because, even if
+         * bfqq does not start to be served immediately, the resulting
+         * delay for bfqq's I/O is however lower or much lower than
+         * the ideal completion time to be guaranteed to bfqq's I/O.
+         *
+         * In both cases, preemption is needed only if, according to
+         * the timestamps of both bfqq and of the in-service queue,
+         * bfqq actually is the next queue to serve. So, to reduce
+         * useless preemptions, the return value of
+         * next_queue_may_preempt() is considered in the next compound
+         * condition too. Yet next_queue_may_preempt() just checks a
+         * simple, necessary condition for bfqq to be the next queue
+         * to serve. In fact, to evaluate a sufficient condition, the
+         * timestamps of the in-service queue would need to be
+         * updated, and this operation is quite costly (see the
+         * comments on bfq_bfqq_update_budg_for_activation()).
          */
-        if (bfqd->in_service_queue && bfqq_wants_to_preempt &&
-            bfqd->in_service_queue->wr_coeff < bfqq->wr_coeff &&
+        if (bfqd->in_service_queue &&
+            ((bfqq_wants_to_preempt &&
+              bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) ||
+             bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue)) &&
             next_queue_may_preempt(bfqd))
                 bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
                                 false, BFQQE_PREEMPTED);
 }
 
+static void bfq_reset_inject_limit(struct bfq_data *bfqd,
+                                   struct bfq_queue *bfqq)
+{
+        /* invalidate baseline total service time */
+        bfqq->last_serv_time_ns = 0;
+
+        /*
+         * Reset pointer in case we are waiting for
+         * some request completion.
+         */
+        bfqd->waited_rq = NULL;
+
+        /*
+         * If bfqq has a short think time, then start by setting the
+         * inject limit to 0 prudentially, because the service time of
+         * an injected I/O request may be higher than the think time
+         * of bfqq, and therefore, if one request was injected when
+         * bfqq remains empty, this injected request might delay the
+         * service of the next I/O request for bfqq significantly. In
+         * case bfqq can actually tolerate some injection, then the
+         * adaptive update will however raise the limit soon. This
+         * lucky circumstance holds exactly because bfqq has a short
+         * think time, and thus, after remaining empty, is likely to
+         * get new I/O enqueued---and then completed---before being
+         * expired. This is the very pattern that gives the
+         * limit-update algorithm the chance to measure the effect of
+         * injection on request service times, and then to update the
+         * limit accordingly.
+         *
+         * However, in the following special case, the inject limit is
+         * left to 1 even if the think time is short: bfqq's I/O is
+         * synchronized with that of some other queue, i.e., bfqq may
+         * receive new I/O only after the I/O of the other queue is
+         * completed. Keeping the inject limit to 1 allows the
+         * blocking I/O to be served while bfqq is in service. And
+         * this is very convenient both for bfqq and for overall
+         * throughput, as explained in detail in the comments in
+         * bfq_update_has_short_ttime().
+         *
+         * On the opposite end, if bfqq has a long think time, then
+         * start directly by 1, because:
+         * a) on the bright side, keeping at most one request in
+         * service in the drive is unlikely to cause any harm to the
+         * latency of bfqq's requests, as the service time of a single
+         * request is likely to be lower than the think time of bfqq;
+         * b) on the downside, after becoming empty, bfqq is likely to
+         * expire before getting its next request. With this request
+         * arrival pattern, it is very hard to sample total service
+         * times and update the inject limit accordingly (see comments
+         * on bfq_update_inject_limit()). So the limit is likely to be
+         * never, or at least seldom, updated.  As a consequence, by
+         * setting the limit to 1, we avoid that no injection ever
+         * occurs with bfqq. On the downside, this proactive step
+         * further reduces chances to actually compute the baseline
+         * total service time. Thus it reduces chances to execute the
+         * limit-update algorithm and possibly raise the limit to more
+         * than 1.
+         */
+        if (bfq_bfqq_has_short_ttime(bfqq))
+                bfqq->inject_limit = 0;
+        else
+                bfqq->inject_limit = 1;
+
+        bfqq->decrease_time_jif = jiffies;
+}
+
 static void bfq_add_request(struct request *rq)
 {
         struct bfq_queue *bfqq = RQ_BFQQ(rq);
@@ -1749,77 +1871,119 @@ static void bfq_add_request(struct request *rq)
 
         if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_sync(bfqq)) {
                 /*
+                 * Detect whether bfqq's I/O seems synchronized with
+                 * that of some other queue, i.e., whether bfqq, after
+                 * remaining empty, happens to receive new I/O only
+                 * right after some I/O request of the other queue has
+                 * been completed. We call waker queue the other
+                 * queue, and we assume, for simplicity, that bfqq may
+                 * have at most one waker queue.
+                 *
+                 * A remarkable throughput boost can be reached by
+                 * unconditionally injecting the I/O of the waker
+                 * queue, every time a new bfq_dispatch_request
+                 * happens to be invoked while I/O is being plugged
+                 * for bfqq.  In addition to boosting throughput, this
+                 * unblocks bfqq's I/O, thereby improving bandwidth
+                 * and latency for bfqq. Note that these same results
+                 * may be achieved with the general injection
+                 * mechanism, but less effectively. For details on
+                 * this aspect, see the comments on the choice of the
+                 * queue for injection in bfq_select_queue().
+                 *
+                 * Turning back to the detection of a waker queue, a
+                 * queue Q is deemed as a waker queue for bfqq if, for
+                 * two consecutive times, bfqq happens to become non
+                 * empty right after a request of Q has been
+                 * completed. In particular, on the first time, Q is
+                 * tentatively set as a candidate waker queue, while
+                 * on the second time, the flag
+                 * bfq_bfqq_has_waker(bfqq) is set to confirm that Q
+                 * is a waker queue for bfqq. These detection steps
+                 * are performed only if bfqq has a long think time,
+                 * so as to make it more likely that bfqq's I/O is
+                 * actually being blocked by a synchronization. This
+                 * last filter, plus the above two-times requirement,
+                 * make false positives less likely.
+                 *
+                 * NOTE
+                 *
+                 * The sooner a waker queue is detected, the sooner
+                 * throughput can be boosted by injecting I/O from the
+                 * waker queue. Fortunately, detection is likely to be
+                 * actually fast, for the following reasons. While
+                 * blocked by synchronization, bfqq has a long think
+                 * time. This implies that bfqq's inject limit is at
+                 * least equal to 1 (see the comments in
+                 * bfq_update_inject_limit()). So, thanks to
+                 * injection, the waker queue is likely to be served
+                 * during the very first I/O-plugging time interval
+                 * for bfqq. This triggers the first step of the
+                 * detection mechanism. Thanks again to injection, the
+                 * candidate waker queue is then likely to be
+                 * confirmed no later than during the next
+                 * I/O-plugging interval for bfqq.
+                 */
+                if (!bfq_bfqq_has_short_ttime(bfqq) &&
+                    ktime_get_ns() - bfqd->last_completion <
+                    200 * NSEC_PER_USEC) {
+                        if (bfqd->last_completed_rq_bfqq != bfqq &&
+                                   bfqd->last_completed_rq_bfqq !=
+                                   bfqq->waker_bfqq) {
+                                /*
+                                 * First synchronization detected with
+                                 * a candidate waker queue, or with a
+                                 * different candidate waker queue
+                                 * from the current one.
+                                 */
+                                bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq;
+
+                                /*
+                                 * If the waker queue disappears, then
+                                 * bfqq->waker_bfqq must be reset. To
+                                 * this goal, we maintain in each
+                                 * waker queue a list, woken_list, of
+                                 * all the queues that reference the
+                                 * waker queue through their
+                                 * waker_bfqq pointer. When the waker
+                                 * queue exits, the waker_bfqq pointer
+                                 * of all the queues in the woken_list
+                                 * is reset.
+                                 *
+                                 * In addition, if bfqq is already in
+                                 * the woken_list of a waker queue,
+                                 * then, before being inserted into
+                                 * the woken_list of a new waker
+                                 * queue, bfqq must be removed from
+                                 * the woken_list of the old waker
+                                 * queue.
+                                 */
+                                if (!hlist_unhashed(&bfqq->woken_list_node))
+                                        hlist_del_init(&bfqq->woken_list_node);
+                                hlist_add_head(&bfqq->woken_list_node,
+                                    &bfqd->last_completed_rq_bfqq->woken_list);
+
+                                bfq_clear_bfqq_has_waker(bfqq);
+                        } else if (bfqd->last_completed_rq_bfqq ==
+                                   bfqq->waker_bfqq &&
+                                   !bfq_bfqq_has_waker(bfqq)) {
+                                /*
+                                 * synchronization with waker_bfqq
+                                 * seen for the second time
+                                 */
+                                bfq_mark_bfqq_has_waker(bfqq);
+                        }
+                }
+
+                /*
                  * Periodically reset inject limit, to make sure that
                  * the latter eventually drops in case workload
                  * changes, see step (3) in the comments on
                  * bfq_update_inject_limit().
                  */
                 if (time_is_before_eq_jiffies(bfqq->decrease_time_jif +
-                                             msecs_to_jiffies(1000))) {
-                        /* invalidate baseline total service time */
-                        bfqq->last_serv_time_ns = 0;
-
-                        /*
-                         * Reset pointer in case we are waiting for
-                         * some request completion.
-                         */
-                        bfqd->waited_rq = NULL;
-
-                        /*
-                         * If bfqq has a short think time, then start
-                         * by setting the inject limit to 0
-                         * prudentially, because the service time of
-                         * an injected I/O request may be higher than
-                         * the think time of bfqq, and therefore, if
-                         * one request was injected when bfqq remains
-                         * empty, this injected request might delay
-                         * the service of the next I/O request for
-                         * bfqq significantly. In case bfqq can
-                         * actually tolerate some injection, then the
-                         * adaptive update will however raise the
-                         * limit soon. This lucky circumstance holds
-                         * exactly because bfqq has a short think
-                         * time, and thus, after remaining empty, is
-                         * likely to get new I/O enqueued---and then
-                         * completed---before being expired. This is
-                         * the very pattern that gives the
-                         * limit-update algorithm the chance to
-                         * measure the effect of injection on request
-                         * service times, and then to update the limit
-                         * accordingly.
-                         *
-                         * On the opposite end, if bfqq has a long
-                         * think time, then start directly by 1,
-                         * because:
-                         * a) on the bright side, keeping at most one
-                         * request in service in the drive is unlikely
-                         * to cause any harm to the latency of bfqq's
-                         * requests, as the service time of a single
-                         * request is likely to be lower than the
-                         * think time of bfqq;
-                         * b) on the downside, after becoming empty,
-                         * bfqq is likely to expire before getting its
-                         * next request. With this request arrival
-                         * pattern, it is very hard to sample total
-                         * service times and update the inject limit
-                         * accordingly (see comments on
-                         * bfq_update_inject_limit()). So the limit is
-                         * likely to be never, or at least seldom,
-                         * updated.  As a consequence, by setting the
-                         * limit to 1, we avoid that no injection ever
-                         * occurs with bfqq. On the downside, this
-                         * proactive step further reduces chances to
-                         * actually compute the baseline total service
-                         * time. Thus it reduces chances to execute the
-                         * limit-update algorithm and possibly raise the
-                         * limit to more than 1.
-                         */
-                        if (bfq_bfqq_has_short_ttime(bfqq))
-                                bfqq->inject_limit = 0;
-                        else
-                                bfqq->inject_limit = 1;
-                        bfqq->decrease_time_jif = jiffies;
-                }
+                                             msecs_to_jiffies(1000)))
+                        bfq_reset_inject_limit(bfqd, bfqq);
 
                 /*
                  * The following conditions must hold to setup a new
@@ -2027,7 +2191,8 @@ static void bfq_remove_request(struct request_queue *q,
 
 }
 
-static bool bfq_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
+static bool bfq_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio,
+                unsigned int nr_segs)
 {
         struct request_queue *q = hctx->queue;
         struct bfq_data *bfqd = q->elevator->elevator_data;
@@ -2050,7 +2215,7 @@ static bool bfq_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
                 bfqd->bio_bfqq = NULL;
         bfqd->bio_bic = bic;
 
-        ret = blk_mq_sched_try_merge(q, bio, &free);
+        ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free);
 
         if (free)
                 blk_mq_free_request(free);
@@ -2513,6 +2678,7 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
                  * to enjoy weight raising if split soon.
                  */
                 bic->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff;
+                bic->saved_wr_start_at_switch_to_srt = bfq_smallest_from_now();
                 bic->saved_wr_cur_max_time = bfq_wr_duration(bfqq->bfqd);
                 bic->saved_last_wr_start_finish = jiffies;
         } else {
@@ -3045,7 +3211,186 @@ static void bfq_dispatch_remove(struct request_queue *q, struct request *rq)
         bfq_remove_request(q, rq);
 }
 
-static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
+/*
+ * There is a case where idling does not have to be performed for
+ * throughput concerns, but to preserve the throughput share of
+ * the process associated with bfqq.
+ *
+ * To introduce this case, we can note that allowing the drive
+ * to enqueue more than one request at a time, and hence
+ * delegating de facto final scheduling decisions to the
+ * drive's internal scheduler, entails loss of control on the
+ * actual request service order. In particular, the critical
+ * situation is when requests from different processes happen
+ * to be present, at the same time, in the internal queue(s)
+ * of the drive. In such a situation, the drive, by deciding
+ * the service order of the internally-queued requests, does
+ * determine also the actual throughput distribution among
+ * these processes. But the drive typically has no notion or
+ * concern about per-process throughput distribution, and
+ * makes its decisions only on a per-request basis. Therefore,
+ * the service distribution enforced by the drive's internal
+ * scheduler is likely to coincide with the desired throughput
+ * distribution only in a completely symmetric, or favorably
+ * skewed scenario where:
+ * (i-a) each of these processes must get the same throughput as
+ *       the others,
+ * (i-b) in case (i-a) does not hold, it holds that the process
+ *       associated with bfqq must receive a lower or equal
+ *       throughput than any of the other processes;
+ * (ii)  the I/O of each process has the same properties, in
+ *       terms of locality (sequential or random), direction
+ *       (reads or writes), request sizes, greediness
+ *       (from I/O-bound to sporadic), and so on;
+
+ * In fact, in such a scenario, the drive tends to treat the requests
+ * of each process in about the same way as the requests of the
+ * others, and thus to provide each of these processes with about the
+ * same throughput.  This is exactly the desired throughput
+ * distribution if (i-a) holds, or, if (i-b) holds instead, this is an
+ * even more convenient distribution for (the process associated with)
+ * bfqq.
+ *
+ * In contrast, in any asymmetric or unfavorable scenario, device
+ * idling (I/O-dispatch plugging) is certainly needed to guarantee
+ * that bfqq receives its assigned fraction of the device throughput
+ * (see [1] for details).
+ *
+ * The problem is that idling may significantly reduce throughput with
+ * certain combinations of types of I/O and devices. An important
+ * example is sync random I/O on flash storage with command
+ * queueing. So, unless bfqq falls in cases where idling also boosts
+ * throughput, it is important to check conditions (i-a), i(-b) and
+ * (ii) accurately, so as to avoid idling when not strictly needed for
+ * service guarantees.
+ *
+ * Unfortunately, it is extremely difficult to thoroughly check
+ * condition (ii). And, in case there are active groups, it becomes
+ * very difficult to check conditions (i-a) and (i-b) too.  In fact,
+ * if there are active groups, then, for conditions (i-a) or (i-b) to
+ * become false 'indirectly', it is enough that an active group
+ * contains more active processes or sub-groups than some other active
+ * group. More precisely, for conditions (i-a) or (i-b) to become
+ * false because of such a group, it is not even necessary that the
+ * group is (still) active: it is sufficient that, even if the group
+ * has become inactive, some of its descendant processes still have
+ * some request already dispatched but still waiting for
+ * completion. In fact, requests have still to be guaranteed their
+ * share of the throughput even after being dispatched. In this
+ * respect, it is easy to show that, if a group frequently becomes
+ * inactive while still having in-flight requests, and if, when this
+ * happens, the group is not considered in the calculation of whether
+ * the scenario is asymmetric, then the group may fail to be
+ * guaranteed its fair share of the throughput (basically because
+ * idling may not be performed for the descendant processes of the
+ * group, but it had to be).  We address this issue with the following
+ * bi-modal behavior, implemented in the function
+ * bfq_asymmetric_scenario().
+ *
+ * If there are groups with requests waiting for completion
+ * (as commented above, some of these groups may even be
+ * already inactive), then the scenario is tagged as
+ * asymmetric, conservatively, without checking any of the
+ * conditions (i-a), (i-b) or (ii). So the device is idled for bfqq.
+ * This behavior matches also the fact that groups are created
+ * exactly if controlling I/O is a primary concern (to
+ * preserve bandwidth and latency guarantees).
+ *
+ * On the opposite end, if there are no groups with requests waiting
+ * for completion, then only conditions (i-a) and (i-b) are actually
+ * controlled, i.e., provided that conditions (i-a) or (i-b) holds,
+ * idling is not performed, regardless of whether condition (ii)
+ * holds.  In other words, only if conditions (i-a) and (i-b) do not
+ * hold, then idling is allowed, and the device tends to be prevented
+ * from queueing many requests, possibly of several processes. Since
+ * there are no groups with requests waiting for completion, then, to
+ * control conditions (i-a) and (i-b) it is enough to check just
+ * whether all the queues with requests waiting for completion also
+ * have the same weight.
+ *
+ * Not checking condition (ii) evidently exposes bfqq to the
+ * risk of getting less throughput than its fair share.
+ * However, for queues with the same weight, a further
+ * mechanism, preemption, mitigates or even eliminates this
+ * problem. And it does so without consequences on overall
+ * throughput. This mechanism and its benefits are explained
+ * in the next three paragraphs.
+ *
+ * Even if a queue, say Q, is expired when it remains idle, Q
+ * can still preempt the new in-service queue if the next
+ * request of Q arrives soon (see the comments on
+ * bfq_bfqq_update_budg_for_activation). If all queues and
+ * groups have the same weight, this form of preemption,
+ * combined with the hole-recovery heuristic described in the
+ * comments on function bfq_bfqq_update_budg_for_activation,
+ * are enough to preserve a correct bandwidth distribution in
+ * the mid term, even without idling. In fact, even if not
+ * idling allows the internal queues of the device to contain
+ * many requests, and thus to reorder requests, we can rather
+ * safely assume that the internal scheduler still preserves a
+ * minimum of mid-term fairness.
+ *
+ * More precisely, this preemption-based, idleless approach
+ * provides fairness in terms of IOPS, and not sectors per
+ * second. This can be seen with a simple example. Suppose
+ * that there are two queues with the same weight, but that
+ * the first queue receives requests of 8 sectors, while the
+ * second queue receives requests of 1024 sectors. In
+ * addition, suppose that each of the two queues contains at
+ * most one request at a time, which implies that each queue
+ * always remains idle after it is served. Finally, after
+ * remaining idle, each queue receives very quickly a new
+ * request. It follows that the two queues are served
+ * alternatively, preempting each other if needed. This
+ * implies that, although both queues have the same weight,
+ * the queue with large requests receives a service that is
+ * 1024/8 times as high as the service received by the other
+ * queue.
+ *
+ * The motivation for using preemption instead of idling (for
+ * queues with the same weight) is that, by not idling,
+ * service guarantees are preserved (completely or at least in
+ * part) without minimally sacrificing throughput. And, if
+ * there is no active group, then the primary expectation for
+ * this device is probably a high throughput.
+ *
+ * We are now left only with explaining the additional
+ * compound condition that is checked below for deciding
+ * whether the scenario is asymmetric. To explain this
+ * compound condition, we need to add that the function
+ * bfq_asymmetric_scenario checks the weights of only
+ * non-weight-raised queues, for efficiency reasons (see
+ * comments on bfq_weights_tree_add()). Then the fact that
+ * bfqq is weight-raised is checked explicitly here. More
+ * precisely, the compound condition below takes into account
+ * also the fact that, even if bfqq is being weight-raised,
+ * the scenario is still symmetric if all queues with requests
+ * waiting for completion happen to be
+ * weight-raised. Actually, we should be even more precise
+ * here, and differentiate between interactive weight raising
+ * and soft real-time weight raising.
+ *
+ * As a side note, it is worth considering that the above
+ * device-idling countermeasures may however fail in the
+ * following unlucky scenario: if idling is (correctly)
+ * disabled in a time period during which all symmetry
+ * sub-conditions hold, and hence the device is allowed to
+ * enqueue many requests, but at some later point in time some
+ * sub-condition stops to hold, then it may become impossible
+ * to let requests be served in the desired order until all
+ * the requests already queued in the device have been served.
+ */
+static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd,
+                                                 struct bfq_queue *bfqq)
+{
+        return (bfqq->wr_coeff > 1 &&
+                bfqd->wr_busy_queues <
+                bfq_tot_busy_queues(bfqd)) ||
+                bfq_asymmetric_scenario(bfqd, bfqq);
+}
+
+static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
+                              enum bfqq_expiration reason)
 {
         /*
          * If this bfqq is shared between multiple processes, check
@@ -3056,7 +3401,22 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
         if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq))
                 bfq_mark_bfqq_split_coop(bfqq);
 
-        if (RB_EMPTY_ROOT(&bfqq->sort_list)) {
+        /*
+         * Consider queues with a higher finish virtual time than
+         * bfqq. If idling_needed_for_service_guarantees(bfqq) returns
+         * true, then bfqq's bandwidth would be violated if an
+         * uncontrolled amount of I/O from these queues were
+         * dispatched while bfqq is waiting for its new I/O to
+         * arrive. This is exactly what may happen if this is a forced
+         * expiration caused by a preemption attempt, and if bfqq is
+         * not re-scheduled. To prevent this from happening, re-queue
+         * bfqq if it needs I/O-dispatch plugging, even if it is
+         * empty. By doing so, bfqq is granted to be served before the
+         * above queues (provided that bfqq is of course eligible).
+         */
+        if (RB_EMPTY_ROOT(&bfqq->sort_list) &&
+            !(reason == BFQQE_PREEMPTED &&
+              idling_needed_for_service_guarantees(bfqd, bfqq))) {
                 if (bfqq->dispatched == 0)
                         /*
                          * Overloading budget_timeout field to store
@@ -3073,7 +3433,8 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq)
                  * Resort priority tree of potential close cooperators.
                  * See comments on bfq_pos_tree_add_move() for the unlikely().
                  */
-                if (unlikely(!bfqd->nonrot_with_queueing))
+                if (unlikely(!bfqd->nonrot_with_queueing &&
+                             !RB_EMPTY_ROOT(&bfqq->sort_list)))
                         bfq_pos_tree_add_move(bfqd, bfqq);
         }
 
@@ -3574,7 +3935,7 @@ void bfq_bfqq_expire(struct bfq_data *bfqd,
          * reason.
          */
         __bfq_bfqq_recalc_budget(bfqd, bfqq, reason);
-        if (__bfq_bfqq_expire(bfqd, bfqq))
+        if (__bfq_bfqq_expire(bfqd, bfqq, reason))
                 /* bfqq is gone, no more actions on it */
                 return;
 
@@ -3721,184 +4082,6 @@ static bool idling_boosts_thr_without_issues(struct bfq_data *bfqd,
 }
 
 /*
- * There is a case where idling does not have to be performed for
- * throughput concerns, but to preserve the throughput share of
- * the process associated with bfqq.
- *
- * To introduce this case, we can note that allowing the drive
- * to enqueue more than one request at a time, and hence
- * delegating de facto final scheduling decisions to the
- * drive's internal scheduler, entails loss of control on the
- * actual request service order. In particular, the critical
- * situation is when requests from different processes happen
- * to be present, at the same time, in the internal queue(s)
- * of the drive. In such a situation, the drive, by deciding
- * the service order of the internally-queued requests, does
- * determine also the actual throughput distribution among
- * these processes. But the drive typically has no notion or
- * concern about per-process throughput distribution, and
- * makes its decisions only on a per-request basis. Therefore,
- * the service distribution enforced by the drive's internal
- * scheduler is likely to coincide with the desired throughput
- * distribution only in a completely symmetric, or favorably
- * skewed scenario where:
- * (i-a) each of these processes must get the same throughput as
- *       the others,
- * (i-b) in case (i-a) does not hold, it holds that the process
- *       associated with bfqq must receive a lower or equal
- *       throughput than any of the other processes;
- * (ii)  the I/O of each process has the same properties, in
- *       terms of locality (sequential or random), direction
- *       (reads or writes), request sizes, greediness
- *       (from I/O-bound to sporadic), and so on;
-
- * In fact, in such a scenario, the drive tends to treat the requests
- * of each process in about the same way as the requests of the
- * others, and thus to provide each of these processes with about the
- * same throughput.  This is exactly the desired throughput
- * distribution if (i-a) holds, or, if (i-b) holds instead, this is an
- * even more convenient distribution for (the process associated with)
- * bfqq.
- *
- * In contrast, in any asymmetric or unfavorable scenario, device
- * idling (I/O-dispatch plugging) is certainly needed to guarantee
- * that bfqq receives its assigned fraction of the device throughput
- * (see [1] for details).
- *
- * The problem is that idling may significantly reduce throughput with
- * certain combinations of types of I/O and devices. An important
- * example is sync random I/O on flash storage with command
- * queueing. So, unless bfqq falls in cases where idling also boosts
- * throughput, it is important to check conditions (i-a), i(-b) and
- * (ii) accurately, so as to avoid idling when not strictly needed for
- * service guarantees.
- *
- * Unfortunately, it is extremely difficult to thoroughly check
- * condition (ii). And, in case there are active groups, it becomes
- * very difficult to check conditions (i-a) and (i-b) too.  In fact,
- * if there are active groups, then, for conditions (i-a) or (i-b) to
- * become false 'indirectly', it is enough that an active group
- * contains more active processes or sub-groups than some other active
- * group. More precisely, for conditions (i-a) or (i-b) to become
- * false because of such a group, it is not even necessary that the
- * group is (still) active: it is sufficient that, even if the group
- * has become inactive, some of its descendant processes still have
- * some request already dispatched but still waiting for
- * completion. In fact, requests have still to be guaranteed their
- * share of the throughput even after being dispatched. In this
- * respect, it is easy to show that, if a group frequently becomes
- * inactive while still having in-flight requests, and if, when this
- * happens, the group is not considered in the calculation of whether
- * the scenario is asymmetric, then the group may fail to be
- * guaranteed its fair share of the throughput (basically because
- * idling may not be performed for the descendant processes of the
- * group, but it had to be).  We address this issue with the following
- * bi-modal behavior, implemented in the function
- * bfq_asymmetric_scenario().
- *
- * If there are groups with requests waiting for completion
- * (as commented above, some of these groups may even be
- * already inactive), then the scenario is tagged as
- * asymmetric, conservatively, without checking any of the
- * conditions (i-a), (i-b) or (ii). So the device is idled for bfqq.
- * This behavior matches also the fact that groups are created
- * exactly if controlling I/O is a primary concern (to
- * preserve bandwidth and latency guarantees).
- *
- * On the opposite end, if there are no groups with requests waiting
- * for completion, then only conditions (i-a) and (i-b) are actually
- * controlled, i.e., provided that conditions (i-a) or (i-b) holds,
- * idling is not performed, regardless of whether condition (ii)
- * holds.  In other words, only if conditions (i-a) and (i-b) do not
- * hold, then idling is allowed, and the device tends to be prevented
- * from queueing many requests, possibly of several processes. Since
- * there are no groups with requests waiting for completion, then, to
- * control conditions (i-a) and (i-b) it is enough to check just
- * whether all the queues with requests waiting for completion also
- * have the same weight.
- *
- * Not checking condition (ii) evidently exposes bfqq to the
- * risk of getting less throughput than its fair share.
- * However, for queues with the same weight, a further
- * mechanism, preemption, mitigates or even eliminates this
- * problem. And it does so without consequences on overall
- * throughput. This mechanism and its benefits are explained
- * in the next three paragraphs.
- *
- * Even if a queue, say Q, is expired when it remains idle, Q
- * can still preempt the new in-service queue if the next
- * request of Q arrives soon (see the comments on
- * bfq_bfqq_update_budg_for_activation). If all queues and
- * groups have the same weight, this form of preemption,
- * combined with the hole-recovery heuristic described in the
- * comments on function bfq_bfqq_update_budg_for_activation,
- * are enough to preserve a correct bandwidth distribution in
- * the mid term, even without idling. In fact, even if not
- * idling allows the internal queues of the device to contain
- * many requests, and thus to reorder requests, we can rather
- * safely assume that the internal scheduler still preserves a
- * minimum of mid-term fairness.
- *
- * More precisely, this preemption-based, idleless approach
- * provides fairness in terms of IOPS, and not sectors per
- * second. This can be seen with a simple example. Suppose
- * that there are two queues with the same weight, but that
- * the first queue receives requests of 8 sectors, while the
- * second queue receives requests of 1024 sectors. In
- * addition, suppose that each of the two queues contains at
- * most one request at a time, which implies that each queue
- * always remains idle after it is served. Finally, after
- * remaining idle, each queue receives very quickly a new
- * request. It follows that the two queues are served
- * alternatively, preempting each other if needed. This
- * implies that, although both queues have the same weight,
- * the queue with large requests receives a service that is
- * 1024/8 times as high as the service received by the other
- * queue.
- *
- * The motivation for using preemption instead of idling (for
- * queues with the same weight) is that, by not idling,
- * service guarantees are preserved (completely or at least in
- * part) without minimally sacrificing throughput. And, if
- * there is no active group, then the primary expectation for
- * this device is probably a high throughput.
- *
- * We are now left only with explaining the additional
- * compound condition that is checked below for deciding
- * whether the scenario is asymmetric. To explain this
- * compound condition, we need to add that the function
- * bfq_asymmetric_scenario checks the weights of only
- * non-weight-raised queues, for efficiency reasons (see
- * comments on bfq_weights_tree_add()). Then the fact that
- * bfqq is weight-raised is checked explicitly here. More
- * precisely, the compound condition below takes into account
- * also the fact that, even if bfqq is being weight-raised,
- * the scenario is still symmetric if all queues with requests
- * waiting for completion happen to be
- * weight-raised. Actually, we should be even more precise
- * here, and differentiate between interactive weight raising
- * and soft real-time weight raising.
- *
- * As a side note, it is worth considering that the above
- * device-idling countermeasures may however fail in the
- * following unlucky scenario: if idling is (correctly)
- * disabled in a time period during which all symmetry
- * sub-conditions hold, and hence the device is allowed to
- * enqueue many requests, but at some later point in time some
- * sub-condition stops to hold, then it may become impossible
- * to let requests be served in the desired order until all
- * the requests already queued in the device have been served.
- */
-static bool idling_needed_for_service_guarantees(struct bfq_data *bfqd,
-                                                 struct bfq_queue *bfqq)
-{
-        return (bfqq->wr_coeff > 1 &&
-                bfqd->wr_busy_queues <
-                bfq_tot_busy_queues(bfqd)) ||
-                bfq_asymmetric_scenario(bfqd, bfqq);
-}
-
-/*
  * For a queue that becomes empty, device idling is allowed only if
  * this function returns true for that queue. As a consequence, since
  * device idling plays a critical role for both throughput boosting
@@ -4156,22 +4339,95 @@ check_queue:
             (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) {
                 struct bfq_queue *async_bfqq =
                         bfqq->bic && bfqq->bic->bfqq[0] &&
-                        bfq_bfqq_busy(bfqq->bic->bfqq[0]) ?
+                        bfq_bfqq_busy(bfqq->bic->bfqq[0]) &&
+                        bfqq->bic->bfqq[0]->next_rq ?
                         bfqq->bic->bfqq[0] : NULL;
 
                 /*
-                 * If the process associated with bfqq has also async
-                 * I/O pending, then inject it
-                 * unconditionally. Injecting I/O from the same
-                 * process can cause no harm to the process. On the
-                 * contrary, it can only increase bandwidth and reduce
-                 * latency for the process.
+                 * The next three mutually-exclusive ifs decide
+                 * whether to try injection, and choose the queue to
+                 * pick an I/O request from.
+                 *
+                 * The first if checks whether the process associated
+                 * with bfqq has also async I/O pending. If so, it
+                 * injects such I/O unconditionally. Injecting async
+                 * I/O from the same process can cause no harm to the
+                 * process. On the contrary, it can only increase
+                 * bandwidth and reduce latency for the process.
+                 *
+                 * The second if checks whether there happens to be a
+                 * non-empty waker queue for bfqq, i.e., a queue whose
+                 * I/O needs to be completed for bfqq to receive new
+                 * I/O. This happens, e.g., if bfqq is associated with
+                 * a process that does some sync. A sync generates
+                 * extra blocking I/O, which must be completed before
+                 * the process associated with bfqq can go on with its
+                 * I/O. If the I/O of the waker queue is not served,
+                 * then bfqq remains empty, and no I/O is dispatched,
+                 * until the idle timeout fires for bfqq. This is
+                 * likely to result in lower bandwidth and higher
+                 * latencies for bfqq, and in a severe loss of total
+                 * throughput. The best action to take is therefore to
+                 * serve the waker queue as soon as possible. So do it
+                 * (without relying on the third alternative below for
+                 * eventually serving waker_bfqq's I/O; see the last
+                 * paragraph for further details). This systematic
+                 * injection of I/O from the waker queue does not
+                 * cause any delay to bfqq's I/O. On the contrary,
+                 * next bfqq's I/O is brought forward dramatically,
+                 * for it is not blocked for milliseconds.
+                 *
+                 * The third if checks whether bfqq is a queue for
+                 * which it is better to avoid injection. It is so if
+                 * bfqq delivers more throughput when served without
+                 * any further I/O from other queues in the middle, or
+                 * if the service times of bfqq's I/O requests both
+                 * count more than overall throughput, and may be
+                 * easily increased by injection (this happens if bfqq
+                 * has a short think time). If none of these
+                 * conditions holds, then a candidate queue for
+                 * injection is looked for through
+                 * bfq_choose_bfqq_for_injection(). Note that the
+                 * latter may return NULL (for example if the inject
+                 * limit for bfqq is currently 0).
+                 *
+                 * NOTE: motivation for the second alternative
+                 *
+                 * Thanks to the way the inject limit is updated in
+                 * bfq_update_has_short_ttime(), it is rather likely
+                 * that, if I/O is being plugged for bfqq and the
+                 * waker queue has pending I/O requests that are
+                 * blocking bfqq's I/O, then the third alternative
+                 * above lets the waker queue get served before the
+                 * I/O-plugging timeout fires. So one may deem the
+                 * second alternative superfluous. It is not, because
+                 * the third alternative may be way less effective in
+                 * case of a synchronization. For two main
+                 * reasons. First, throughput may be low because the
+                 * inject limit may be too low to guarantee the same
+                 * amount of injected I/O, from the waker queue or
+                 * other queues, that the second alternative
+                 * guarantees (the second alternative unconditionally
+                 * injects a pending I/O request of the waker queue
+                 * for each bfq_dispatch_request()). Second, with the
+                 * third alternative, the duration of the plugging,
+                 * i.e., the time before bfqq finally receives new I/O,
+                 * may not be minimized, because the waker queue may
+                 * happen to be served only after other queues.
                  */
                 if (async_bfqq &&
                     icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&
                     bfq_serv_to_charge(async_bfqq->next_rq, async_bfqq) <=
                     bfq_bfqq_budget_left(async_bfqq))
                         bfqq = bfqq->bic->bfqq[0];
+                else if (bfq_bfqq_has_waker(bfqq) &&
+                           bfq_bfqq_busy(bfqq->waker_bfqq) &&
+                           bfqq->next_rq &&
+                           bfq_serv_to_charge(bfqq->waker_bfqq->next_rq,
+                                              bfqq->waker_bfqq) <=
+                           bfq_bfqq_budget_left(bfqq->waker_bfqq)
+                        )
+                        bfqq = bfqq->waker_bfqq;
                 else if (!idling_boosts_thr_without_issues(bfqd, bfqq) &&
                          (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 ||
                           !bfq_bfqq_has_short_ttime(bfqq)))
@@ -4403,7 +4659,7 @@ exit:
         return rq;
 }
 
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
 static void bfq_update_dispatch_stats(struct request_queue *q,
                                       struct request *rq,
                                       struct bfq_queue *in_serv_queue,
@@ -4453,7 +4709,7 @@ static inline void bfq_update_dispatch_stats(struct request_queue *q,
                                              struct request *rq,
                                              struct bfq_queue *in_serv_queue,
                                              bool idle_timer_disabled) {}
-#endif
+#endif /* CONFIG_BFQ_CGROUP_DEBUG */
 
 static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
 {
@@ -4560,8 +4816,11 @@ static void bfq_put_cooperator(struct bfq_queue *bfqq)
 
 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 {
+        struct bfq_queue *item;
+        struct hlist_node *n;
+
         if (bfqq == bfqd->in_service_queue) {
-                __bfq_bfqq_expire(bfqd, bfqq);
+                __bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT);
                 bfq_schedule_dispatch(bfqd);
         }
 
@@ -4569,6 +4828,18 @@ static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 
         bfq_put_cooperator(bfqq);
 
+        /* remove bfqq from woken list */
+        if (!hlist_unhashed(&bfqq->woken_list_node))
+                hlist_del_init(&bfqq->woken_list_node);
+
+        /* reset waker for all queues in woken list */
+        hlist_for_each_entry_safe(item, n, &bfqq->woken_list,
+                                  woken_list_node) {
+                item->waker_bfqq = NULL;
+                bfq_clear_bfqq_has_waker(item);
+                hlist_del_init(&item->woken_list_node);
+        }
+
         bfq_put_queue(bfqq); /* release process reference */
 }
 
@@ -4584,6 +4855,7 @@ static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync)
                 unsigned long flags;
 
                 spin_lock_irqsave(&bfqd->lock, flags);
+                bfqq->bic = NULL;
                 bfq_exit_bfqq(bfqd, bfqq);
                 bic_set_bfqq(bic, NULL, is_sync);
                 spin_unlock_irqrestore(&bfqd->lock, flags);
@@ -4687,6 +4959,8 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
         RB_CLEAR_NODE(&bfqq->entity.rb_node);
         INIT_LIST_HEAD(&bfqq->fifo);
         INIT_HLIST_NODE(&bfqq->burst_list_node);
+        INIT_HLIST_NODE(&bfqq->woken_list_node);
+        INIT_HLIST_HEAD(&bfqq->woken_list);
 
         bfqq->ref = 0;
         bfqq->bfqd = bfqd;
@@ -4854,7 +5128,7 @@ static void bfq_update_has_short_ttime(struct bfq_data *bfqd,
                                        struct bfq_queue *bfqq,
                                        struct bfq_io_cq *bic)
 {
-        bool has_short_ttime = true;
+        bool has_short_ttime = true, state_changed;
 
         /*
          * No need to update has_short_ttime if bfqq is async or in
@@ -4879,13 +5153,102 @@ static void bfq_update_has_short_ttime(struct bfq_data *bfqd,
              bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle))
                 has_short_ttime = false;
 
-        bfq_log_bfqq(bfqd, bfqq, "update_has_short_ttime: has_short_ttime %d",
-                     has_short_ttime);
+        state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq);
 
         if (has_short_ttime)
                 bfq_mark_bfqq_has_short_ttime(bfqq);
         else
                 bfq_clear_bfqq_has_short_ttime(bfqq);
+
+        /*
+         * Until the base value for the total service time gets
+         * finally computed for bfqq, the inject limit does depend on
+         * the think-time state (short|long). In particular, the limit
+         * is 0 or 1 if the think time is deemed, respectively, as
+         * short or long (details in the comments in
+         * bfq_update_inject_limit()). Accordingly, the next
+         * instructions reset the inject limit if the think-time state
+         * has changed and the above base value is still to be
+         * computed.
+         *
+         * However, the reset is performed only if more than 100 ms
+         * have elapsed since the last update of the inject limit, or
+         * (inclusive) if the change is from short to long think
+         * time. The reason for this waiting is as follows.
+         *
+         * bfqq may have a long think time because of a
+         * synchronization with some other queue, i.e., because the
+         * I/O of some other queue may need to be completed for bfqq
+         * to receive new I/O. Details in the comments on the choice
+         * of the queue for injection in bfq_select_queue().
+         *
+         * As stressed in those comments, if such a synchronization is
+         * actually in place, then, without injection on bfqq, the
+         * blocking I/O cannot happen to served while bfqq is in
+         * service. As a consequence, if bfqq is granted
+         * I/O-dispatch-plugging, then bfqq remains empty, and no I/O
+         * is dispatched, until the idle timeout fires. This is likely
+         * to result in lower bandwidth and higher latencies for bfqq,
+         * and in a severe loss of total throughput.
+         *
+         * On the opposite end, a non-zero inject limit may allow the
+         * I/O that blocks bfqq to be executed soon, and therefore
+         * bfqq to receive new I/O soon.
+         *
+         * But, if the blocking gets actually eliminated, then the
+         * next think-time sample for bfqq may be very low. This in
+         * turn may cause bfqq's think time to be deemed
+         * short. Without the 100 ms barrier, this new state change
+         * would cause the body of the next if to be executed
+         * immediately. But this would set to 0 the inject
+         * limit. Without injection, the blocking I/O would cause the
+         * think time of bfqq to become long again, and therefore the
+         * inject limit to be raised again, and so on. The only effect
+         * of such a steady oscillation between the two think-time
+         * states would be to prevent effective injection on bfqq.
+         *
+         * In contrast, if the inject limit is not reset during such a
+         * long time interval as 100 ms, then the number of short
+         * think time samples can grow significantly before the reset
+         * is performed. As a consequence, the think time state can
+         * become stable before the reset. Therefore there will be no
+         * state change when the 100 ms elapse, and no reset of the
+         * inject limit. The inject limit remains steadily equal to 1
+         * both during and after the 100 ms. So injection can be
+         * performed at all times, and throughput gets boosted.
+         *
+         * An inject limit equal to 1 is however in conflict, in
+         * general, with the fact that the think time of bfqq is
+         * short, because injection may be likely to delay bfqq's I/O
+         * (as explained in the comments in
+         * bfq_update_inject_limit()). But this does not happen in
+         * this special case, because bfqq's low think time is due to
+         * an effective handling of a synchronization, through
+         * injection. In this special case, bfqq's I/O does not get
+         * delayed by injection; on the contrary, bfqq's I/O is
+         * brought forward, because it is not blocked for
+         * milliseconds.
+         *
+         * In addition, serving the blocking I/O much sooner, and much
+         * more frequently than once per I/O-plugging timeout, makes
+         * it much quicker to detect a waker queue (the concept of
+         * waker queue is defined in the comments in
+         * bfq_add_request()). This makes it possible to start sooner
+         * to boost throughput more effectively, by injecting the I/O
+         * of the waker queue unconditionally on every
+         * bfq_dispatch_request().
+         *
+         * One last, important benefit of not resetting the inject
+         * limit before 100 ms is that, during this time interval, the
+         * base value for the total service time is likely to get
+         * finally computed for bfqq, freeing the inject limit from
+         * its relation with the think time.
+         */
+        if (state_changed && bfqq->last_serv_time_ns == 0 &&
+            (time_is_before_eq_jiffies(bfqq->decrease_time_jif +
+                                      msecs_to_jiffies(100)) ||
+             !has_short_ttime))
+                bfq_reset_inject_limit(bfqd, bfqq);
 }
 
 /*
@@ -4895,19 +5258,9 @@ static void bfq_update_has_short_ttime(struct bfq_data *bfqd,
 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq,
                             struct request *rq)
 {
-        struct bfq_io_cq *bic = RQ_BIC(rq);
-
         if (rq->cmd_flags & REQ_META)
                 bfqq->meta_pending++;
 
-        bfq_update_io_thinktime(bfqd, bfqq);
-        bfq_update_has_short_ttime(bfqd, bfqq, bic);
-        bfq_update_io_seektime(bfqd, bfqq, rq);
-
-        bfq_log_bfqq(bfqd, bfqq,
-                     "rq_enqueued: has_short_ttime=%d (seeky %d)",
-                     bfq_bfqq_has_short_ttime(bfqq), BFQQ_SEEKY(bfqq));
-
         bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
 
         if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) {
@@ -4995,6 +5348,10 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq)
                 bfqq = new_bfqq;
         }
 
+        bfq_update_io_thinktime(bfqd, bfqq);
+        bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq));
+        bfq_update_io_seektime(bfqd, bfqq, rq);
+
         waiting = bfqq && bfq_bfqq_wait_request(bfqq);
         bfq_add_request(rq);
         idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq);
@@ -5007,7 +5364,7 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq)
         return idle_timer_disabled;
 }
 
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
 static void bfq_update_insert_stats(struct request_queue *q,
                                     struct bfq_queue *bfqq,
                                     bool idle_timer_disabled,
@@ -5037,7 +5394,7 @@ static inline void bfq_update_insert_stats(struct request_queue *q,
                                            struct bfq_queue *bfqq,
                                            bool idle_timer_disabled,
                                            unsigned int cmd_flags) {}
-#endif
+#endif /* CONFIG_BFQ_CGROUP_DEBUG */
 
 static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
                                bool at_head)
@@ -5200,6 +5557,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd)
                         1UL<<(BFQ_RATE_SHIFT - 10))
                 bfq_update_rate_reset(bfqd, NULL);
         bfqd->last_completion = now_ns;
+        bfqd->last_completed_rq_bfqq = bfqq;
 
         /*
          * If we are waiting to discover whether the request pattern
@@ -5397,8 +5755,14 @@ static void bfq_update_inject_limit(struct bfq_data *bfqd,
          * total service time, and there seem to be the right
          * conditions to do it, or we can lower the last base value
          * computed.
+         *
+         * NOTE: (bfqd->rq_in_driver == 1) means that there is no I/O
+         * request in flight, because this function is in the code
+         * path that handles the completion of a request of bfqq, and,
+         * in particular, this function is executed before
+         * bfqd->rq_in_driver is decremented in such a code path.
          */
-        if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 0) ||
+        if ((bfqq->last_serv_time_ns == 0 && bfqd->rq_in_driver == 1) ||
             tot_time_ns < bfqq->last_serv_time_ns) {
                 bfqq->last_serv_time_ns = tot_time_ns;
                 /*
@@ -5406,7 +5770,18 @@ static void bfq_update_inject_limit(struct bfq_data *bfqd,
                  * start trying injection.
                  */
                 bfqq->inject_limit = max_t(unsigned int, 1, old_limit);
-        }
+        } else if (!bfqd->rqs_injected && bfqd->rq_in_driver == 1)
+                /*
+                 * No I/O injected and no request still in service in
+                 * the drive: these are the exact conditions for
+                 * computing the base value of the total service time
+                 * for bfqq. So let's update this value, because it is
+                 * rather variable. For example, it varies if the size
+                 * or the spatial locality of the I/O requests in bfqq
+                 * change.
+                 */
+                bfqq->last_serv_time_ns = tot_time_ns;
+
 
         /* update complete, not waiting for any request completion any longer */
         bfqd->waited_rq = NULL;
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index c2faa77824f8..e80adf822bbe 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -357,6 +357,24 @@ struct bfq_queue {
 
         /* max service rate measured so far */
         u32 max_service_rate;
+
+        /*
+         * Pointer to the waker queue for this queue, i.e., to the
+         * queue Q such that this queue happens to get new I/O right
+         * after some I/O request of Q is completed. For details, see
+         * the comments on the choice of the queue for injection in
+         * bfq_select_queue().
+         */
+        struct bfq_queue *waker_bfqq;
+        /* node for woken_list, see below */
+        struct hlist_node woken_list_node;
+        /*
+         * Head of the list of the woken queues for this queue, i.e.,
+         * of the list of the queues for which this queue is a waker
+         * queue. This list is used to reset the waker_bfqq pointer in
+         * the woken queues when this queue exits.
+         */
+        struct hlist_head woken_list;
 };
 
 /**
@@ -533,6 +551,9 @@ struct bfq_data {
         /* time of last request completion (ns) */
         u64 last_completion;
 
+        /* bfqq owning the last completed rq */
+        struct bfq_queue *last_completed_rq_bfqq;
+
         /* time of last transition from empty to non-empty (ns) */
         u64 last_empty_occupied_ns;
 
@@ -743,7 +764,8 @@ enum bfqq_state_flags {
                                  * update
                                  */
         BFQQF_coop,             /* bfqq is shared */
-        BFQQF_split_coop        /* shared bfqq will be split */
+        BFQQF_split_coop,       /* shared bfqq will be split */
+        BFQQF_has_waker         /* bfqq has a waker queue */
 };
 
 #define BFQ_BFQQ_FNS(name)                                              \
@@ -763,6 +785,7 @@ BFQ_BFQQ_FNS(in_large_burst);
 BFQ_BFQQ_FNS(coop);
 BFQ_BFQQ_FNS(split_coop);
 BFQ_BFQQ_FNS(softrt_update);
+BFQ_BFQQ_FNS(has_waker);
 #undef BFQ_BFQQ_FNS
 
 /* Expiration reasons. */
@@ -777,8 +800,13 @@ enum bfqq_expiration {
         BFQQE_PREEMPTED         /* preemption in progress */
 };
 
+struct bfq_stat {
+        struct percpu_counter           cpu_cnt;
+        atomic64_t                      aux_cnt;
+};
+
 struct bfqg_stats {
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
+#ifdef CONFIG_BFQ_CGROUP_DEBUG
         /* number of ios merged */
         struct blkg_rwstat              merged;
         /* total time spent on device in ns, may not be accurate w/ queueing */
@@ -788,25 +816,25 @@ struct bfqg_stats {
         /* number of IOs queued up */
         struct blkg_rwstat              queued;
         /* total disk time and nr sectors dispatched by this group */
-        struct blkg_stat                time;
+        struct bfq_stat         time;
         /* sum of number of ios queued across all samples */
-        struct blkg_stat                avg_queue_size_sum;
+        struct bfq_stat         avg_queue_size_sum;
         /* count of samples taken for average */
-        struct blkg_stat                avg_queue_size_samples;
+        struct bfq_stat         avg_queue_size_samples;
         /* how many times this group has been removed from service tree */
-        struct blkg_stat                dequeue;
+        struct bfq_stat         dequeue;
         /* total time spent waiting for it to be assigned a timeslice. */
-        struct blkg_stat                group_wait_time;
+        struct bfq_stat         group_wait_time;
         /* time spent idling for this blkcg_gq */
-        struct blkg_stat                idle_time;
+        struct bfq_stat         idle_time;
         /* total time with empty current active q with other requests queued */
-        struct blkg_stat                empty_time;
+        struct bfq_stat         empty_time;
         /* fields after this shouldn't be cleared on stat reset */
         u64                             start_group_wait_time;
         u64                             start_idle_time;
         u64                             start_empty_time;
         uint16_t                        flags;
-#endif  /* CONFIG_BFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
+#endif /* CONFIG_BFQ_CGROUP_DEBUG */
 };
 
 #ifdef CONFIG_BFQ_GROUP_IOSCHED
diff --git a/block/bio.c b/block/bio.c
index ce797d73bb43..29cd6cf4da51 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -558,14 +558,6 @@ void bio_put(struct bio *bio)
 }
 EXPORT_SYMBOL(bio_put);
 
-int bio_phys_segments(struct request_queue *q, struct bio *bio)
-{
-        if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
-                blk_recount_segments(q, bio);
-
-        return bio->bi_phys_segments;
-}
-
 /**
  *      __bio_clone_fast - clone a bio that shares the original bio's biovec
  *      @bio: destination bio
@@ -731,10 +723,10 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,
                 }
         }
 
-        if (bio_full(bio))
+        if (bio_full(bio, len))
                 return 0;
 
-        if (bio->bi_phys_segments >= queue_max_segments(q))
+        if (bio->bi_vcnt >= queue_max_segments(q))
                 return 0;
 
         bvec = &bio->bi_io_vec[bio->bi_vcnt];
@@ -744,8 +736,6 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,
         bio->bi_vcnt++;
  done:
         bio->bi_iter.bi_size += len;
-        bio->bi_phys_segments = bio->bi_vcnt;
-        bio_set_flag(bio, BIO_SEG_VALID);
         return len;
 }
 
@@ -807,7 +797,7 @@ void __bio_add_page(struct bio *bio, struct page *page,
         struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 
         WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
-        WARN_ON_ONCE(bio_full(bio));
+        WARN_ON_ONCE(bio_full(bio, len));
 
         bv->bv_page = page;
         bv->bv_offset = off;
@@ -834,7 +824,7 @@ int bio_add_page(struct bio *bio, struct page *page,
         bool same_page = false;
 
         if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) {
-                if (bio_full(bio))
+                if (bio_full(bio, len))
                         return 0;
                 __bio_add_page(bio, page, len, offset);
         }
@@ -842,22 +832,19 @@ int bio_add_page(struct bio *bio, struct page *page,
 }
 EXPORT_SYMBOL(bio_add_page);
 
-static void bio_get_pages(struct bio *bio)
+void bio_release_pages(struct bio *bio, bool mark_dirty)
 {
         struct bvec_iter_all iter_all;
         struct bio_vec *bvec;
 
-        bio_for_each_segment_all(bvec, bio, iter_all)
-                get_page(bvec->bv_page);
-}
-
-static void bio_release_pages(struct bio *bio)
-{
-        struct bvec_iter_all iter_all;
-        struct bio_vec *bvec;
+        if (bio_flagged(bio, BIO_NO_PAGE_REF))
+                return;
 
-        bio_for_each_segment_all(bvec, bio, iter_all)
+        bio_for_each_segment_all(bvec, bio, iter_all) {
+                if (mark_dirty && !PageCompound(bvec->bv_page))
+                        set_page_dirty_lock(bvec->bv_page);
                 put_page(bvec->bv_page);
+        }
 }
 
 static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
@@ -922,7 +909,7 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
                         if (same_page)
                                 put_page(page);
                 } else {
-                        if (WARN_ON_ONCE(bio_full(bio)))
+                        if (WARN_ON_ONCE(bio_full(bio, len)))
                                 return -EINVAL;
                         __bio_add_page(bio, page, len, offset);
                 }
@@ -966,13 +953,10 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
                         ret = __bio_iov_bvec_add_pages(bio, iter);
                 else
                         ret = __bio_iov_iter_get_pages(bio, iter);
-        } while (!ret && iov_iter_count(iter) && !bio_full(bio));
+        } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
 
-        if (iov_iter_bvec_no_ref(iter))
+        if (is_bvec)
                 bio_set_flag(bio, BIO_NO_PAGE_REF);
-        else if (is_bvec)
-                bio_get_pages(bio);
-
         return bio->bi_vcnt ? 0 : ret;
 }
 
@@ -1124,8 +1108,7 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,
         if (data->nr_segs > UIO_MAXIOV)
                 return NULL;
 
-        bmd = kmalloc(sizeof(struct bio_map_data) +
-                       sizeof(struct iovec) * data->nr_segs, gfp_mask);
+        bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask);
         if (!bmd)
                 return NULL;
         memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);
@@ -1371,8 +1354,6 @@ struct bio *bio_map_user_iov(struct request_queue *q,
         int j;
         struct bio *bio;
         int ret;
-        struct bio_vec *bvec;
-        struct bvec_iter_all iter_all;
 
         if (!iov_iter_count(iter))
                 return ERR_PTR(-EINVAL);
@@ -1439,31 +1420,11 @@ struct bio *bio_map_user_iov(struct request_queue *q,
         return bio;
 
  out_unmap:
-        bio_for_each_segment_all(bvec, bio, iter_all) {
-                put_page(bvec->bv_page);
-        }
+        bio_release_pages(bio, false);
         bio_put(bio);
         return ERR_PTR(ret);
 }
 
-static void __bio_unmap_user(struct bio *bio)
-{
-        struct bio_vec *bvec;
-        struct bvec_iter_all iter_all;
-
-        /*
-         * make sure we dirty pages we wrote to
-         */
-        bio_for_each_segment_all(bvec, bio, iter_all) {
-                if (bio_data_dir(bio) == READ)
-                        set_page_dirty_lock(bvec->bv_page);
-
-                put_page(bvec->bv_page);
-        }
-
-        bio_put(bio);
-}
-
 /**
  *      bio_unmap_user  -       unmap a bio
  *      @bio:           the bio being unmapped
@@ -1475,7 +1436,8 @@ static void __bio_unmap_user(struct bio *bio)
  */
 void bio_unmap_user(struct bio *bio)
 {
-        __bio_unmap_user(bio);
+        bio_release_pages(bio, bio_data_dir(bio) == READ);
+        bio_put(bio);
         bio_put(bio);
 }
 
@@ -1695,9 +1657,7 @@ static void bio_dirty_fn(struct work_struct *work)
         while ((bio = next) != NULL) {
                 next = bio->bi_private;
 
-                bio_set_pages_dirty(bio);
-                if (!bio_flagged(bio, BIO_NO_PAGE_REF))
-                        bio_release_pages(bio);
+                bio_release_pages(bio, true);
                 bio_put(bio);
         }
 }
@@ -1713,8 +1673,7 @@ void bio_check_pages_dirty(struct bio *bio)
                         goto defer;
         }
 
-        if (!bio_flagged(bio, BIO_NO_PAGE_REF))
-                bio_release_pages(bio);
+        bio_release_pages(bio, false);
         bio_put(bio);
         return;
 defer:
@@ -1775,18 +1734,6 @@ void generic_end_io_acct(struct request_queue *q, int req_op,
 }
 EXPORT_SYMBOL(generic_end_io_acct);
 
-#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
-void bio_flush_dcache_pages(struct bio *bi)
-{
-        struct bio_vec bvec;
-        struct bvec_iter iter;
-
-        bio_for_each_segment(bvec, bi, iter)
-                flush_dcache_page(bvec.bv_page);
-}
-EXPORT_SYMBOL(bio_flush_dcache_pages);
-#endif
-
 static inline bool bio_remaining_done(struct bio *bio)
 {
         /*
@@ -1914,10 +1861,7 @@ void bio_trim(struct bio *bio, int offset, int size)
         if (offset == 0 && size == bio->bi_iter.bi_size)
                 return;
 
-        bio_clear_flag(bio, BIO_SEG_VALID);
-
         bio_advance(bio, offset << 9);
-
         bio->bi_iter.bi_size = size;
 
         if (bio_integrity(bio))
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 1f7127b03490..53b7bd4c7000 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -79,6 +79,7 @@ static void blkg_free(struct blkcg_gq *blkg)
 
         blkg_rwstat_exit(&blkg->stat_ios);
         blkg_rwstat_exit(&blkg->stat_bytes);
+        percpu_ref_exit(&blkg->refcnt);
         kfree(blkg);
 }
 
@@ -86,8 +87,6 @@ static void __blkg_release(struct rcu_head *rcu)
 {
         struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
 
-        percpu_ref_exit(&blkg->refcnt);
-
         /* release the blkcg and parent blkg refs this blkg has been holding */
         css_put(&blkg->blkcg->css);
         if (blkg->parent)
@@ -132,6 +131,9 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
         if (!blkg)
                 return NULL;
 
+        if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
+                goto err_free;
+
         if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
             blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
                 goto err_free;
@@ -244,11 +246,6 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
                 blkg_get(blkg->parent);
         }
 
-        ret = percpu_ref_init(&blkg->refcnt, blkg_release, 0,
-                              GFP_NOWAIT | __GFP_NOWARN);
-        if (ret)
-                goto err_cancel_ref;
-
         /* invoke per-policy init */
         for (i = 0; i < BLKCG_MAX_POLS; i++) {
                 struct blkcg_policy *pol = blkcg_policy[i];
@@ -281,8 +278,6 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
         blkg_put(blkg);
         return ERR_PTR(ret);
 
-err_cancel_ref:
-        percpu_ref_exit(&blkg->refcnt);
 err_put_congested:
         wb_congested_put(wb_congested);
 err_put_css:
@@ -549,7 +544,7 @@ EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
  * Print @rwstat to @sf for the device assocaited with @pd.
  */
 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
-                         const struct blkg_rwstat *rwstat)
+                         const struct blkg_rwstat_sample *rwstat)
 {
         static const char *rwstr[] = {
                 [BLKG_RWSTAT_READ]      = "Read",
@@ -567,31 +562,17 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 
         for (i = 0; i < BLKG_RWSTAT_NR; i++)
                 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
-                           (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
+                           rwstat->cnt[i]);
 
-        v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
-                atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]) +
-                atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_DISCARD]);
-        seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
+        v = rwstat->cnt[BLKG_RWSTAT_READ] +
+                rwstat->cnt[BLKG_RWSTAT_WRITE] +
+                rwstat->cnt[BLKG_RWSTAT_DISCARD];
+        seq_printf(sf, "%s Total %llu\n", dname, v);
         return v;
 }
 EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
 
 /**
- * blkg_prfill_stat - prfill callback for blkg_stat
- * @sf: seq_file to print to
- * @pd: policy private data of interest
- * @off: offset to the blkg_stat in @pd
- *
- * prfill callback for printing a blkg_stat.
- */
-u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
-{
-        return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
-}
-EXPORT_SYMBOL_GPL(blkg_prfill_stat);
-
-/**
  * blkg_prfill_rwstat - prfill callback for blkg_rwstat
  * @sf: seq_file to print to
  * @pd: policy private data of interest
@@ -602,8 +583,9 @@ EXPORT_SYMBOL_GPL(blkg_prfill_stat);
 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
                        int off)
 {
-        struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
+        struct blkg_rwstat_sample rwstat = { };
 
+        blkg_rwstat_read((void *)pd + off, &rwstat);
         return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
@@ -611,8 +593,9 @@ EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
 static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
                                     struct blkg_policy_data *pd, int off)
 {
-        struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
+        struct blkg_rwstat_sample rwstat = { };
 
+        blkg_rwstat_read((void *)pd->blkg + off, &rwstat);
         return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 
@@ -654,8 +637,9 @@ static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
                                               struct blkg_policy_data *pd,
                                               int off)
 {
-        struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
-                                                              NULL, off);
+        struct blkg_rwstat_sample rwstat;
+
+        blkg_rwstat_recursive_sum(pd->blkg, NULL, off, &rwstat);
         return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 
@@ -690,52 +674,11 @@ int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
 EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
 
 /**
- * blkg_stat_recursive_sum - collect hierarchical blkg_stat
- * @blkg: blkg of interest
- * @pol: blkcg_policy which contains the blkg_stat
- * @off: offset to the blkg_stat in blkg_policy_data or @blkg
- *
- * Collect the blkg_stat specified by @blkg, @pol and @off and all its
- * online descendants and their aux counts.  The caller must be holding the
- * queue lock for online tests.
- *
- * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
- * at @off bytes into @blkg's blkg_policy_data of the policy.
- */
-u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
-                            struct blkcg_policy *pol, int off)
-{
-        struct blkcg_gq *pos_blkg;
-        struct cgroup_subsys_state *pos_css;
-        u64 sum = 0;
-
-        lockdep_assert_held(&blkg->q->queue_lock);
-
-        rcu_read_lock();
-        blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
-                struct blkg_stat *stat;
-
-                if (!pos_blkg->online)
-                        continue;
-
-                if (pol)
-                        stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
-                else
-                        stat = (void *)blkg + off;
-
-                sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
-        }
-        rcu_read_unlock();
-
-        return sum;
-}
-EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
-
-/**
  * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
  * @blkg: blkg of interest
  * @pol: blkcg_policy which contains the blkg_rwstat
  * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
+ * @sum: blkg_rwstat_sample structure containing the results
  *
  * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
  * online descendants and their aux counts.  The caller must be holding the
@@ -744,13 +687,12 @@ EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
  * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
  * is at @off bytes into @blkg's blkg_policy_data of the policy.
  */
-struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
-                                             struct blkcg_policy *pol, int off)
+void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
+                int off, struct blkg_rwstat_sample *sum)
 {
         struct blkcg_gq *pos_blkg;
         struct cgroup_subsys_state *pos_css;
-        struct blkg_rwstat sum = { };
-        int i;
+        unsigned int i;
 
         lockdep_assert_held(&blkg->q->queue_lock);
 
@@ -767,13 +709,9 @@ struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
                         rwstat = (void *)pos_blkg + off;
 
                 for (i = 0; i < BLKG_RWSTAT_NR; i++)
-                        atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
-                                percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
-                                &sum.aux_cnt[i]);
+                        sum->cnt[i] = blkg_rwstat_read_counter(rwstat, i);
         }
         rcu_read_unlock();
-
-        return sum;
 }
 EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
 
@@ -939,7 +877,7 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
         hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
                 const char *dname;
                 char *buf;
-                struct blkg_rwstat rwstat;
+                struct blkg_rwstat_sample rwstat;
                 u64 rbytes, wbytes, rios, wios, dbytes, dios;
                 size_t size = seq_get_buf(sf, &buf), off = 0;
                 int i;
@@ -959,17 +897,17 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
 
                 spin_lock_irq(&blkg->q->queue_lock);
 
-                rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
-                                        offsetof(struct blkcg_gq, stat_bytes));
-                rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
-                wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
-                dbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
+                blkg_rwstat_recursive_sum(blkg, NULL,
+                                offsetof(struct blkcg_gq, stat_bytes), &rwstat);
+                rbytes = rwstat.cnt[BLKG_RWSTAT_READ];
+                wbytes = rwstat.cnt[BLKG_RWSTAT_WRITE];
+                dbytes = rwstat.cnt[BLKG_RWSTAT_DISCARD];
 
-                rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
-                                        offsetof(struct blkcg_gq, stat_ios));
-                rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
-                wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
-                dios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
+                blkg_rwstat_recursive_sum(blkg, NULL,
+                                        offsetof(struct blkcg_gq, stat_ios), &rwstat);
+                rios = rwstat.cnt[BLKG_RWSTAT_READ];
+                wios = rwstat.cnt[BLKG_RWSTAT_WRITE];
+                dios = rwstat.cnt[BLKG_RWSTAT_DISCARD];
 
                 spin_unlock_irq(&blkg->q->queue_lock);
 
@@ -1006,8 +944,12 @@ static int blkcg_print_stat(struct seq_file *sf, void *v)
                 }
 next:
                 if (has_stats) {
-                        off += scnprintf(buf+off, size-off, "\n");
-                        seq_commit(sf, off);
+                        if (off < size - 1) {
+                                off += scnprintf(buf+off, size-off, "\n");
+                                seq_commit(sf, off);
+                        } else {
+                                seq_commit(sf, -1);
+                        }
                 }
         }
 
@@ -1391,7 +1333,8 @@ pd_prealloc:
 
         spin_lock_irq(&q->queue_lock);
 
-        list_for_each_entry(blkg, &q->blkg_list, q_node) {
+        /* blkg_list is pushed at the head, reverse walk to init parents first */
+        list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
                 struct blkg_policy_data *pd;
 
                 if (blkg->pd[pol->plid])
diff --git a/block/blk-core.c b/block/blk-core.c
index 8340f69670d8..5d1fc8e17dd1 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -120,6 +120,42 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
 }
 EXPORT_SYMBOL(blk_rq_init);
 
+#define REQ_OP_NAME(name) [REQ_OP_##name] = #name
+static const char *const blk_op_name[] = {
+        REQ_OP_NAME(READ),
+        REQ_OP_NAME(WRITE),
+        REQ_OP_NAME(FLUSH),
+        REQ_OP_NAME(DISCARD),
+        REQ_OP_NAME(SECURE_ERASE),
+        REQ_OP_NAME(ZONE_RESET),
+        REQ_OP_NAME(WRITE_SAME),
+        REQ_OP_NAME(WRITE_ZEROES),
+        REQ_OP_NAME(SCSI_IN),
+        REQ_OP_NAME(SCSI_OUT),
+        REQ_OP_NAME(DRV_IN),
+        REQ_OP_NAME(DRV_OUT),
+};
+#undef REQ_OP_NAME
+
+/**
+ * blk_op_str - Return string XXX in the REQ_OP_XXX.
+ * @op: REQ_OP_XXX.
+ *
+ * Description: Centralize block layer function to convert REQ_OP_XXX into
+ * string format. Useful in the debugging and tracing bio or request. For
+ * invalid REQ_OP_XXX it returns string "UNKNOWN".
+ */
+inline const char *blk_op_str(unsigned int op)
+{
+        const char *op_str = "UNKNOWN";
+
+        if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
+                op_str = blk_op_name[op];
+
+        return op_str;
+}
+EXPORT_SYMBOL_GPL(blk_op_str);
+
 static const struct {
         int             errno;
         const char      *name;
@@ -167,18 +203,23 @@ int blk_status_to_errno(blk_status_t status)
 }
 EXPORT_SYMBOL_GPL(blk_status_to_errno);
 
-static void print_req_error(struct request *req, blk_status_t status)
+static void print_req_error(struct request *req, blk_status_t status,
+                const char *caller)
 {
         int idx = (__force int)status;
 
         if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
                 return;
 
-        printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu flags %x\n",
-                                __func__, blk_errors[idx].name,
-                                req->rq_disk ?  req->rq_disk->disk_name : "?",
-                                (unsigned long long)blk_rq_pos(req),
-                                req->cmd_flags);
+        printk_ratelimited(KERN_ERR
+                "%s: %s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
+                "phys_seg %u prio class %u\n",
+                caller, blk_errors[idx].name,
+                req->rq_disk ? req->rq_disk->disk_name : "?",
+                blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),
+                req->cmd_flags & ~REQ_OP_MASK,
+                req->nr_phys_segments,
+                IOPRIO_PRIO_CLASS(req->ioprio));
 }
 
 static void req_bio_endio(struct request *rq, struct bio *bio,
@@ -550,15 +591,15 @@ void blk_put_request(struct request *req)
 }
 EXPORT_SYMBOL(blk_put_request);
 
-bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
-                            struct bio *bio)
+bool bio_attempt_back_merge(struct request *req, struct bio *bio,
+                unsigned int nr_segs)
 {
         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
 
-        if (!ll_back_merge_fn(q, req, bio))
+        if (!ll_back_merge_fn(req, bio, nr_segs))
                 return false;
 
-        trace_block_bio_backmerge(q, req, bio);
+        trace_block_bio_backmerge(req->q, req, bio);
 
         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
                 blk_rq_set_mixed_merge(req);
@@ -571,15 +612,15 @@ bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
         return true;
 }
 
-bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
-                             struct bio *bio)
+bool bio_attempt_front_merge(struct request *req, struct bio *bio,
+                unsigned int nr_segs)
 {
         const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
 
-        if (!ll_front_merge_fn(q, req, bio))
+        if (!ll_front_merge_fn(req, bio, nr_segs))
                 return false;
 
-        trace_block_bio_frontmerge(q, req, bio);
+        trace_block_bio_frontmerge(req->q, req, bio);
 
         if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
                 blk_rq_set_mixed_merge(req);
@@ -621,6 +662,7 @@ no_merge:
  * blk_attempt_plug_merge - try to merge with %current's plugged list
  * @q: request_queue new bio is being queued at
  * @bio: new bio being queued
+ * @nr_segs: number of segments in @bio
  * @same_queue_rq: pointer to &struct request that gets filled in when
  * another request associated with @q is found on the plug list
  * (optional, may be %NULL)
@@ -639,7 +681,7 @@ no_merge:
  * Caller must ensure !blk_queue_nomerges(q) beforehand.
  */
 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
-                            struct request **same_queue_rq)
+                unsigned int nr_segs, struct request **same_queue_rq)
 {
         struct blk_plug *plug;
         struct request *rq;
@@ -668,10 +710,10 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
 
                 switch (blk_try_merge(rq, bio)) {
                 case ELEVATOR_BACK_MERGE:
-                        merged = bio_attempt_back_merge(q, rq, bio);
+                        merged = bio_attempt_back_merge(rq, bio, nr_segs);
                         break;
                 case ELEVATOR_FRONT_MERGE:
-                        merged = bio_attempt_front_merge(q, rq, bio);
+                        merged = bio_attempt_front_merge(rq, bio, nr_segs);
                         break;
                 case ELEVATOR_DISCARD_MERGE:
                         merged = bio_attempt_discard_merge(q, rq, bio);
@@ -687,18 +729,6 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
         return false;
 }
 
-void blk_init_request_from_bio(struct request *req, struct bio *bio)
-{
-        if (bio->bi_opf & REQ_RAHEAD)
-                req->cmd_flags |= REQ_FAILFAST_MASK;
-
-        req->__sector = bio->bi_iter.bi_sector;
-        req->ioprio = bio_prio(bio);
-        req->write_hint = bio->bi_write_hint;
-        blk_rq_bio_prep(req->q, req, bio);
-}
-EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
-
 static void handle_bad_sector(struct bio *bio, sector_t maxsector)
 {
         char b[BDEVNAME_SIZE];
@@ -1163,7 +1193,7 @@ static int blk_cloned_rq_check_limits(struct request_queue *q,
          * Recalculate it to check the request correctly on this queue's
          * limitation.
          */
-        blk_recalc_rq_segments(rq);
+        rq->nr_phys_segments = blk_recalc_rq_segments(rq);
         if (rq->nr_phys_segments > queue_max_segments(q)) {
                 printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
                         __func__, rq->nr_phys_segments, queue_max_segments(q));
@@ -1348,7 +1378,7 @@ EXPORT_SYMBOL_GPL(blk_steal_bios);
  *
  *     This special helper function is only for request stacking drivers
  *     (e.g. request-based dm) so that they can handle partial completion.
- *     Actual device drivers should use blk_end_request instead.
+ *     Actual device drivers should use blk_mq_end_request instead.
  *
  *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
  *     %false return from this function.
@@ -1373,7 +1403,7 @@ bool blk_update_request(struct request *req, blk_status_t error,
 
         if (unlikely(error && !blk_rq_is_passthrough(req) &&
                      !(req->rq_flags & RQF_QUIET)))
-                print_req_error(req, error);
+                print_req_error(req, error, __func__);
 
         blk_account_io_completion(req, nr_bytes);
 
@@ -1432,28 +1462,13 @@ bool blk_update_request(struct request *req, blk_status_t error,
                 }
 
                 /* recalculate the number of segments */
-                blk_recalc_rq_segments(req);
+                req->nr_phys_segments = blk_recalc_rq_segments(req);
         }
 
         return true;
 }
 EXPORT_SYMBOL_GPL(blk_update_request);
 
-void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
-                     struct bio *bio)
-{
-        if (bio_has_data(bio))
-                rq->nr_phys_segments = bio_phys_segments(q, bio);
-        else if (bio_op(bio) == REQ_OP_DISCARD)
-                rq->nr_phys_segments = 1;
-
-        rq->__data_len = bio->bi_iter.bi_size;
-        rq->bio = rq->biotail = bio;
-
-        if (bio->bi_disk)
-                rq->rq_disk = bio->bi_disk;
-}
-
 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
 /**
  * rq_flush_dcache_pages - Helper function to flush all pages in a request
diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c
index d22e61bced86..d973c38ee4fd 100644
--- a/block/blk-iolatency.c
+++ b/block/blk-iolatency.c
@@ -618,44 +618,26 @@ static void blkcg_iolatency_done_bio(struct rq_qos *rqos, struct bio *bio)
 
                 inflight = atomic_dec_return(&rqw->inflight);
                 WARN_ON_ONCE(inflight < 0);
-                if (iolat->min_lat_nsec == 0)
-                        goto next;
-                iolatency_record_time(iolat, &bio->bi_issue, now,
-                                      issue_as_root);
-                window_start = atomic64_read(&iolat->window_start);
-                if (now > window_start &&
-                    (now - window_start) >= iolat->cur_win_nsec) {
-                        if (atomic64_cmpxchg(&iolat->window_start,
-                                        window_start, now) == window_start)
-                                iolatency_check_latencies(iolat, now);
+                /*
+                 * If bi_status is BLK_STS_AGAIN, the bio wasn't actually
+                 * submitted, so do not account for it.
+                 */
+                if (iolat->min_lat_nsec && bio->bi_status != BLK_STS_AGAIN) {
+                        iolatency_record_time(iolat, &bio->bi_issue, now,
+                                              issue_as_root);
+                        window_start = atomic64_read(&iolat->window_start);
+                        if (now > window_start &&
+                            (now - window_start) >= iolat->cur_win_nsec) {
+                                if (atomic64_cmpxchg(&iolat->window_start,
+                                             window_start, now) == window_start)
+                                        iolatency_check_latencies(iolat, now);
+                        }
                 }
-next:
                 wake_up(&rqw->wait);
                 blkg = blkg->parent;
         }
 }
 
-static void blkcg_iolatency_cleanup(struct rq_qos *rqos, struct bio *bio)
-{
-        struct blkcg_gq *blkg;
-
-        blkg = bio->bi_blkg;
-        while (blkg && blkg->parent) {
-                struct rq_wait *rqw;
-                struct iolatency_grp *iolat;
-
-                iolat = blkg_to_lat(blkg);
-                if (!iolat)
-                        goto next;
-
-                rqw = &iolat->rq_wait;
-                atomic_dec(&rqw->inflight);
-                wake_up(&rqw->wait);
-next:
-                blkg = blkg->parent;
-        }
-}
-
 static void blkcg_iolatency_exit(struct rq_qos *rqos)
 {
         struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos);
@@ -667,7 +649,6 @@ static void blkcg_iolatency_exit(struct rq_qos *rqos)
 
 static struct rq_qos_ops blkcg_iolatency_ops = {
         .throttle = blkcg_iolatency_throttle,
-        .cleanup = blkcg_iolatency_cleanup,
         .done_bio = blkcg_iolatency_done_bio,
         .exit = blkcg_iolatency_exit,
 };
@@ -778,8 +759,10 @@ static int iolatency_set_min_lat_nsec(struct blkcg_gq *blkg, u64 val)
 
         if (!oldval && val)
                 return 1;
-        if (oldval && !val)
+        if (oldval && !val) {
+                blkcg_clear_delay(blkg);
                 return -1;
+        }
         return 0;
 }
 
diff --git a/block/blk-map.c b/block/blk-map.c
index db9373bd31ac..3a62e471d81b 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -18,13 +18,19 @@
 int blk_rq_append_bio(struct request *rq, struct bio **bio)
 {
         struct bio *orig_bio = *bio;
+        struct bvec_iter iter;
+        struct bio_vec bv;
+        unsigned int nr_segs = 0;
 
         blk_queue_bounce(rq->q, bio);
 
+        bio_for_each_bvec(bv, *bio, iter)
+                nr_segs++;
+
         if (!rq->bio) {
-                blk_rq_bio_prep(rq->q, rq, *bio);
+                blk_rq_bio_prep(rq, *bio, nr_segs);
         } else {
-                if (!ll_back_merge_fn(rq->q, rq, *bio)) {
+                if (!ll_back_merge_fn(rq, *bio, nr_segs)) {
                         if (orig_bio != *bio) {
                                 bio_put(*bio);
                                 *bio = orig_bio;
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 17713d7d98d5..57f7990b342d 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -105,7 +105,7 @@ static struct bio *blk_bio_discard_split(struct request_queue *q,
 static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
                 struct bio *bio, struct bio_set *bs, unsigned *nsegs)
 {
-        *nsegs = 1;
+        *nsegs = 0;
 
         if (!q->limits.max_write_zeroes_sectors)
                 return NULL;
@@ -202,8 +202,6 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
         struct bio_vec bv, bvprv, *bvprvp = NULL;
         struct bvec_iter iter;
         unsigned nsegs = 0, sectors = 0;
-        bool do_split = true;
-        struct bio *new = NULL;
         const unsigned max_sectors = get_max_io_size(q, bio);
         const unsigned max_segs = queue_max_segments(q);
 
@@ -245,45 +243,36 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
                 }
         }
 
-        do_split = false;
+        *segs = nsegs;
+        return NULL;
 split:
         *segs = nsegs;
-
-        if (do_split) {
-                new = bio_split(bio, sectors, GFP_NOIO, bs);
-                if (new)
-                        bio = new;
-        }
-
-        return do_split ? new : NULL;
+        return bio_split(bio, sectors, GFP_NOIO, bs);
 }
 
-void blk_queue_split(struct request_queue *q, struct bio **bio)
+void __blk_queue_split(struct request_queue *q, struct bio **bio,
+                unsigned int *nr_segs)
 {
-        struct bio *split, *res;
-        unsigned nsegs;
+        struct bio *split;
 
         switch (bio_op(*bio)) {
         case REQ_OP_DISCARD:
         case REQ_OP_SECURE_ERASE:
-                split = blk_bio_discard_split(q, *bio, &q->bio_split, &nsegs);
+                split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs);
                 break;
         case REQ_OP_WRITE_ZEROES:
-                split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split, &nsegs);
+                split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split,
+                                nr_segs);
                 break;
         case REQ_OP_WRITE_SAME:
-                split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs);
+                split = blk_bio_write_same_split(q, *bio, &q->bio_split,
+                                nr_segs);
                 break;
         default:
-                split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs);
+                split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);
                 break;
         }
 
-        /* physical segments can be figured out during splitting */
-        res = split ? split : *bio;
-        res->bi_phys_segments = nsegs;
-        bio_set_flag(res, BIO_SEG_VALID);
-
         if (split) {
                 /* there isn't chance to merge the splitted bio */
                 split->bi_opf |= REQ_NOMERGE;
@@ -304,19 +293,25 @@ void blk_queue_split(struct request_queue *q, struct bio **bio)
                 *bio = split;
         }
 }
+
+void blk_queue_split(struct request_queue *q, struct bio **bio)
+{
+        unsigned int nr_segs;
+
+        __blk_queue_split(q, bio, &nr_segs);
+}
 EXPORT_SYMBOL(blk_queue_split);
 
-static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
-                                             struct bio *bio)
+unsigned int blk_recalc_rq_segments(struct request *rq)
 {
         unsigned int nr_phys_segs = 0;
-        struct bvec_iter iter;
+        struct req_iterator iter;
         struct bio_vec bv;
 
-        if (!bio)
+        if (!rq->bio)
                 return 0;
 
-        switch (bio_op(bio)) {
+        switch (bio_op(rq->bio)) {
         case REQ_OP_DISCARD:
         case REQ_OP_SECURE_ERASE:
         case REQ_OP_WRITE_ZEROES:
@@ -325,30 +320,11 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
                 return 1;
         }
 
-        for_each_bio(bio) {
-                bio_for_each_bvec(bv, bio, iter)
-                        bvec_split_segs(q, &bv, &nr_phys_segs, NULL, UINT_MAX);
-        }
-
+        rq_for_each_bvec(bv, rq, iter)
+                bvec_split_segs(rq->q, &bv, &nr_phys_segs, NULL, UINT_MAX);
         return nr_phys_segs;
 }
 
-void blk_recalc_rq_segments(struct request *rq)
-{
-        rq->nr_phys_segments = __blk_recalc_rq_segments(rq->q, rq->bio);
-}
-
-void blk_recount_segments(struct request_queue *q, struct bio *bio)
-{
-        struct bio *nxt = bio->bi_next;
-
-        bio->bi_next = NULL;
-        bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
-        bio->bi_next = nxt;
-
-        bio_set_flag(bio, BIO_SEG_VALID);
-}
-
 static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
                 struct scatterlist *sglist)
 {
@@ -519,16 +495,13 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,
 }
 EXPORT_SYMBOL(blk_rq_map_sg);
 
-static inline int ll_new_hw_segment(struct request_queue *q,
-                                    struct request *req,
-                                    struct bio *bio)
+static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
+                unsigned int nr_phys_segs)
 {
-        int nr_phys_segs = bio_phys_segments(q, bio);
-
-        if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q))
+        if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(req->q))
                 goto no_merge;
 
-        if (blk_integrity_merge_bio(q, req, bio) == false)
+        if (blk_integrity_merge_bio(req->q, req, bio) == false)
                 goto no_merge;
 
         /*
@@ -539,12 +512,11 @@ static inline int ll_new_hw_segment(struct request_queue *q,
         return 1;
 
 no_merge:
-        req_set_nomerge(q, req);
+        req_set_nomerge(req->q, req);
         return 0;
 }
 
-int ll_back_merge_fn(struct request_queue *q, struct request *req,
-                     struct bio *bio)
+int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
 {
         if (req_gap_back_merge(req, bio))
                 return 0;
@@ -553,21 +525,15 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,
                 return 0;
         if (blk_rq_sectors(req) + bio_sectors(bio) >
             blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
-                req_set_nomerge(q, req);
+                req_set_nomerge(req->q, req);
                 return 0;
         }
-        if (!bio_flagged(req->biotail, BIO_SEG_VALID))
-                blk_recount_segments(q, req->biotail);
-        if (!bio_flagged(bio, BIO_SEG_VALID))
-                blk_recount_segments(q, bio);
 
-        return ll_new_hw_segment(q, req, bio);
+        return ll_new_hw_segment(req, bio, nr_segs);
 }
 
-int ll_front_merge_fn(struct request_queue *q, struct request *req,
-                      struct bio *bio)
+int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
 {
-
         if (req_gap_front_merge(req, bio))
                 return 0;
         if (blk_integrity_rq(req) &&
@@ -575,15 +541,11 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req,
                 return 0;
         if (blk_rq_sectors(req) + bio_sectors(bio) >
             blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
-                req_set_nomerge(q, req);
+                req_set_nomerge(req->q, req);
                 return 0;
         }
-        if (!bio_flagged(bio, BIO_SEG_VALID))
-                blk_recount_segments(q, bio);
-        if (!bio_flagged(req->bio, BIO_SEG_VALID))
-                blk_recount_segments(q, req->bio);
 
-        return ll_new_hw_segment(q, req, bio);
+        return ll_new_hw_segment(req, bio, nr_segs);
 }
 
 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index 3afe327f816f..b3f2ba483992 100644
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -17,7 +17,7 @@
 static void print_stat(struct seq_file *m, struct blk_rq_stat *stat)
 {
         if (stat->nr_samples) {
-                seq_printf(m, "samples=%d, mean=%lld, min=%llu, max=%llu",
+                seq_printf(m, "samples=%d, mean=%llu, min=%llu, max=%llu",
                            stat->nr_samples, stat->mean, stat->min, stat->max);
         } else {
                 seq_puts(m, "samples=0");
@@ -29,13 +29,13 @@ static int queue_poll_stat_show(void *data, struct seq_file *m)
         struct request_queue *q = data;
         int bucket;
 
-        for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS/2; bucket++) {
-                seq_printf(m, "read  (%d Bytes): ", 1 << (9+bucket));
-                print_stat(m, &q->poll_stat[2*bucket]);
+        for (bucket = 0; bucket < (BLK_MQ_POLL_STATS_BKTS / 2); bucket++) {
+                seq_printf(m, "read  (%d Bytes): ", 1 << (9 + bucket));
+                print_stat(m, &q->poll_stat[2 * bucket]);
                 seq_puts(m, "\n");
 
-                seq_printf(m, "write (%d Bytes): ",  1 << (9+bucket));
-                print_stat(m, &q->poll_stat[2*bucket+1]);
+                seq_printf(m, "write (%d Bytes): ",  1 << (9 + bucket));
+                print_stat(m, &q->poll_stat[2 * bucket + 1]);
                 seq_puts(m, "\n");
         }
         return 0;
@@ -261,23 +261,6 @@ static int hctx_flags_show(void *data, struct seq_file *m)
         return 0;
 }
 
-#define REQ_OP_NAME(name) [REQ_OP_##name] = #name
-static const char *const op_name[] = {
-        REQ_OP_NAME(READ),
-        REQ_OP_NAME(WRITE),
-        REQ_OP_NAME(FLUSH),
-        REQ_OP_NAME(DISCARD),
-        REQ_OP_NAME(SECURE_ERASE),
-        REQ_OP_NAME(ZONE_RESET),
-        REQ_OP_NAME(WRITE_SAME),
-        REQ_OP_NAME(WRITE_ZEROES),
-        REQ_OP_NAME(SCSI_IN),
-        REQ_OP_NAME(SCSI_OUT),
-        REQ_OP_NAME(DRV_IN),
-        REQ_OP_NAME(DRV_OUT),
-};
-#undef REQ_OP_NAME
-
 #define CMD_FLAG_NAME(name) [__REQ_##name] = #name
 static const char *const cmd_flag_name[] = {
         CMD_FLAG_NAME(FAILFAST_DEV),
@@ -341,13 +324,14 @@ static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state)
 int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
 {
         const struct blk_mq_ops *const mq_ops = rq->q->mq_ops;
-        const unsigned int op = rq->cmd_flags & REQ_OP_MASK;
+        const unsigned int op = req_op(rq);
+        const char *op_str = blk_op_str(op);
 
         seq_printf(m, "%p {.op=", rq);
-        if (op < ARRAY_SIZE(op_name) && op_name[op])
-                seq_printf(m, "%s", op_name[op]);
+        if (strcmp(op_str, "UNKNOWN") == 0)
+                seq_printf(m, "%u", op);
         else
-                seq_printf(m, "%d", op);
+                seq_printf(m, "%s", op_str);
         seq_puts(m, ", .cmd_flags=");
         blk_flags_show(m, rq->cmd_flags & ~REQ_OP_MASK, cmd_flag_name,
                        ARRAY_SIZE(cmd_flag_name));
@@ -779,8 +763,8 @@ static int blk_mq_debugfs_release(struct inode *inode, struct file *file)
 
         if (attr->show)
                 return single_release(inode, file);
-        else
-                return seq_release(inode, file);
+
+        return seq_release(inode, file);
 }
 
 static const struct file_operations blk_mq_debugfs_fops = {
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 2766066a15db..c9d183d6c499 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -224,7 +224,7 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
 }
 
 bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
-                            struct request **merged_request)
+                unsigned int nr_segs, struct request **merged_request)
 {
         struct request *rq;
 
@@ -232,7 +232,7 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
         case ELEVATOR_BACK_MERGE:
                 if (!blk_mq_sched_allow_merge(q, rq, bio))
                         return false;
-                if (!bio_attempt_back_merge(q, rq, bio))
+                if (!bio_attempt_back_merge(rq, bio, nr_segs))
                         return false;
                 *merged_request = attempt_back_merge(q, rq);
                 if (!*merged_request)
@@ -241,7 +241,7 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
         case ELEVATOR_FRONT_MERGE:
                 if (!blk_mq_sched_allow_merge(q, rq, bio))
                         return false;
-                if (!bio_attempt_front_merge(q, rq, bio))
+                if (!bio_attempt_front_merge(rq, bio, nr_segs))
                         return false;
                 *merged_request = attempt_front_merge(q, rq);
                 if (!*merged_request)
@@ -260,7 +260,7 @@ EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
  * of them.
  */
 bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
-                           struct bio *bio)
+                           struct bio *bio, unsigned int nr_segs)
 {
         struct request *rq;
         int checked = 8;
@@ -277,11 +277,13 @@ bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
                 switch (blk_try_merge(rq, bio)) {
                 case ELEVATOR_BACK_MERGE:
                         if (blk_mq_sched_allow_merge(q, rq, bio))
-                                merged = bio_attempt_back_merge(q, rq, bio);
+                                merged = bio_attempt_back_merge(rq, bio,
+                                                nr_segs);
                         break;
                 case ELEVATOR_FRONT_MERGE:
                         if (blk_mq_sched_allow_merge(q, rq, bio))
-                                merged = bio_attempt_front_merge(q, rq, bio);
+                                merged = bio_attempt_front_merge(rq, bio,
+                                                nr_segs);
                         break;
                 case ELEVATOR_DISCARD_MERGE:
                         merged = bio_attempt_discard_merge(q, rq, bio);
@@ -304,13 +306,14 @@ EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge);
  */
 static bool blk_mq_attempt_merge(struct request_queue *q,
                                  struct blk_mq_hw_ctx *hctx,
-                                 struct blk_mq_ctx *ctx, struct bio *bio)
+                                 struct blk_mq_ctx *ctx, struct bio *bio,
+                                 unsigned int nr_segs)
 {
         enum hctx_type type = hctx->type;
 
         lockdep_assert_held(&ctx->lock);
 
-        if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio)) {
+        if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) {
                 ctx->rq_merged++;
                 return true;
         }
@@ -318,7 +321,8 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
         return false;
 }
 
-bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
+bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
+                unsigned int nr_segs)
 {
         struct elevator_queue *e = q->elevator;
         struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
@@ -326,21 +330,18 @@ bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
         bool ret = false;
         enum hctx_type type;
 
-        if (e && e->type->ops.bio_merge) {
-                blk_mq_put_ctx(ctx);
-                return e->type->ops.bio_merge(hctx, bio);
-        }
+        if (e && e->type->ops.bio_merge)
+                return e->type->ops.bio_merge(hctx, bio, nr_segs);
 
         type = hctx->type;
         if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
                         !list_empty_careful(&ctx->rq_lists[type])) {
                 /* default per sw-queue merge */
                 spin_lock(&ctx->lock);
-                ret = blk_mq_attempt_merge(q, hctx, ctx, bio);
+                ret = blk_mq_attempt_merge(q, hctx, ctx, bio, nr_segs);
                 spin_unlock(&ctx->lock);
         }
 
-        blk_mq_put_ctx(ctx);
         return ret;
 }
 
diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h
index 3cf92cbbd8ac..cf22ab00fefb 100644
--- a/block/blk-mq-sched.h
+++ b/block/blk-mq-sched.h
@@ -12,8 +12,9 @@ void blk_mq_sched_assign_ioc(struct request *rq);
 
 void blk_mq_sched_request_inserted(struct request *rq);
 bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
-                                struct request **merged_request);
-bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio);
+                unsigned int nr_segs, struct request **merged_request);
+bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
+                unsigned int nr_segs);
 bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq);
 void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx);
 void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);
@@ -31,12 +32,13 @@ void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e);
 void blk_mq_sched_free_requests(struct request_queue *q);
 
 static inline bool
-blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio)
+blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
+                unsigned int nr_segs)
 {
         if (blk_queue_nomerges(q) || !bio_mergeable(bio))
                 return false;
 
-        return __blk_mq_sched_bio_merge(q, bio);
+        return __blk_mq_sched_bio_merge(q, bio, nr_segs);
 }
 
 static inline bool
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 7513c8eaabee..da19f0bc8876 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -113,7 +113,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
         struct sbq_wait_state *ws;
         DEFINE_SBQ_WAIT(wait);
         unsigned int tag_offset;
-        bool drop_ctx;
         int tag;
 
         if (data->flags & BLK_MQ_REQ_RESERVED) {
@@ -136,7 +135,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
                 return BLK_MQ_TAG_FAIL;
 
         ws = bt_wait_ptr(bt, data->hctx);
-        drop_ctx = data->ctx == NULL;
         do {
                 struct sbitmap_queue *bt_prev;
 
@@ -161,9 +159,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
                 if (tag != -1)
                         break;
 
-                if (data->ctx)
-                        blk_mq_put_ctx(data->ctx);
-
                 bt_prev = bt;
                 io_schedule();
 
@@ -189,9 +184,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
                 ws = bt_wait_ptr(bt, data->hctx);
         } while (1);
 
-        if (drop_ctx && data->ctx)
-                blk_mq_put_ctx(data->ctx);
-
         sbitmap_finish_wait(bt, ws, &wait);
 
 found_tag:
diff --git a/block/blk-mq.c b/block/blk-mq.c
index ce0f5f4ede70..e5ef40c603ca 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -355,13 +355,13 @@ static struct request *blk_mq_get_request(struct request_queue *q,
         struct elevator_queue *e = q->elevator;
         struct request *rq;
         unsigned int tag;
-        bool put_ctx_on_error = false;
+        bool clear_ctx_on_error = false;
 
         blk_queue_enter_live(q);
         data->q = q;
         if (likely(!data->ctx)) {
                 data->ctx = blk_mq_get_ctx(q);
-                put_ctx_on_error = true;
+                clear_ctx_on_error = true;
         }
         if (likely(!data->hctx))
                 data->hctx = blk_mq_map_queue(q, data->cmd_flags,
@@ -387,10 +387,8 @@ static struct request *blk_mq_get_request(struct request_queue *q,
 
         tag = blk_mq_get_tag(data);
         if (tag == BLK_MQ_TAG_FAIL) {
-                if (put_ctx_on_error) {
-                        blk_mq_put_ctx(data->ctx);
+                if (clear_ctx_on_error)
                         data->ctx = NULL;
-                }
                 blk_queue_exit(q);
                 return NULL;
         }
@@ -427,8 +425,6 @@ struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
         if (!rq)
                 return ERR_PTR(-EWOULDBLOCK);
 
-        blk_mq_put_ctx(alloc_data.ctx);
-
         rq->__data_len = 0;
         rq->__sector = (sector_t) -1;
         rq->bio = rq->biotail = NULL;
@@ -1764,9 +1760,15 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
         }
 }
 
-static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
+static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
+                unsigned int nr_segs)
 {
-        blk_init_request_from_bio(rq, bio);
+        if (bio->bi_opf & REQ_RAHEAD)
+                rq->cmd_flags |= REQ_FAILFAST_MASK;
+
+        rq->__sector = bio->bi_iter.bi_sector;
+        rq->write_hint = bio->bi_write_hint;
+        blk_rq_bio_prep(rq, bio, nr_segs);
 
         blk_account_io_start(rq, true);
 }
@@ -1936,20 +1938,20 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
         struct request *rq;
         struct blk_plug *plug;
         struct request *same_queue_rq = NULL;
+        unsigned int nr_segs;
         blk_qc_t cookie;
 
         blk_queue_bounce(q, &bio);
-
-        blk_queue_split(q, &bio);
+        __blk_queue_split(q, &bio, &nr_segs);
 
         if (!bio_integrity_prep(bio))
                 return BLK_QC_T_NONE;
 
         if (!is_flush_fua && !blk_queue_nomerges(q) &&
-            blk_attempt_plug_merge(q, bio, &same_queue_rq))
+            blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq))
                 return BLK_QC_T_NONE;
 
-        if (blk_mq_sched_bio_merge(q, bio))
+        if (blk_mq_sched_bio_merge(q, bio, nr_segs))
                 return BLK_QC_T_NONE;
 
         rq_qos_throttle(q, bio);
@@ -1969,11 +1971,10 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 
         cookie = request_to_qc_t(data.hctx, rq);
 
+        blk_mq_bio_to_request(rq, bio, nr_segs);
+
         plug = current->plug;
         if (unlikely(is_flush_fua)) {
-                blk_mq_put_ctx(data.ctx);
-                blk_mq_bio_to_request(rq, bio);
-
                 /* bypass scheduler for flush rq */
                 blk_insert_flush(rq);
                 blk_mq_run_hw_queue(data.hctx, true);
@@ -1985,9 +1986,6 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
                 unsigned int request_count = plug->rq_count;
                 struct request *last = NULL;
 
-                blk_mq_put_ctx(data.ctx);
-                blk_mq_bio_to_request(rq, bio);
-
                 if (!request_count)
                         trace_block_plug(q);
                 else
@@ -2001,8 +1999,6 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 
                 blk_add_rq_to_plug(plug, rq);
         } else if (plug && !blk_queue_nomerges(q)) {
-                blk_mq_bio_to_request(rq, bio);
-
                 /*
                  * We do limited plugging. If the bio can be merged, do that.
                  * Otherwise the existing request in the plug list will be
@@ -2019,8 +2015,6 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
                 blk_add_rq_to_plug(plug, rq);
                 trace_block_plug(q);
 
-                blk_mq_put_ctx(data.ctx);
-
                 if (same_queue_rq) {
                         data.hctx = same_queue_rq->mq_hctx;
                         trace_block_unplug(q, 1, true);
@@ -2029,12 +2023,8 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
                 }
         } else if ((q->nr_hw_queues > 1 && is_sync) || (!q->elevator &&
                         !data.hctx->dispatch_busy)) {
-                blk_mq_put_ctx(data.ctx);
-                blk_mq_bio_to_request(rq, bio);
                 blk_mq_try_issue_directly(data.hctx, rq, &cookie);
         } else {
-                blk_mq_put_ctx(data.ctx);
-                blk_mq_bio_to_request(rq, bio);
                 blk_mq_sched_insert_request(rq, false, true, true);
         }
 
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 633a5a77ee8b..f4bf5161333e 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -151,12 +151,7 @@ static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
  */
 static inline struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
 {
-        return __blk_mq_get_ctx(q, get_cpu());
-}
-
-static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
-{
-        put_cpu();
+        return __blk_mq_get_ctx(q, raw_smp_processor_id());
 }
 
 struct blk_mq_alloc_data {
diff --git a/block/blk.h b/block/blk.h
index 7814aa207153..de6b2e146d6e 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -51,8 +51,6 @@ struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
                 int node, int cmd_size, gfp_t flags);
 void blk_free_flush_queue(struct blk_flush_queue *q);
 
-void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
-                        struct bio *bio);
 void blk_freeze_queue(struct request_queue *q);
 
 static inline void blk_queue_enter_live(struct request_queue *q)
@@ -101,6 +99,18 @@ static inline bool bvec_gap_to_prev(struct request_queue *q,
         return __bvec_gap_to_prev(q, bprv, offset);
 }
 
+static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio,
+                unsigned int nr_segs)
+{
+        rq->nr_phys_segments = nr_segs;
+        rq->__data_len = bio->bi_iter.bi_size;
+        rq->bio = rq->biotail = bio;
+        rq->ioprio = bio_prio(bio);
+
+        if (bio->bi_disk)
+                rq->rq_disk = bio->bi_disk;
+}
+
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 void blk_flush_integrity(void);
 bool __bio_integrity_endio(struct bio *);
@@ -154,14 +164,14 @@ static inline bool bio_integrity_endio(struct bio *bio)
 unsigned long blk_rq_timeout(unsigned long timeout);
 void blk_add_timer(struct request *req);
 
-bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
-                             struct bio *bio);
-bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
-                            struct bio *bio);
+bool bio_attempt_front_merge(struct request *req, struct bio *bio,
+                unsigned int nr_segs);
+bool bio_attempt_back_merge(struct request *req, struct bio *bio,
+                unsigned int nr_segs);
 bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
                 struct bio *bio);
 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
-                            struct request **same_queue_rq);
+                unsigned int nr_segs, struct request **same_queue_rq);
 
 void blk_account_io_start(struct request *req, bool new_io);
 void blk_account_io_completion(struct request *req, unsigned int bytes);
@@ -202,15 +212,17 @@ static inline int blk_should_fake_timeout(struct request_queue *q)
 }
 #endif
 
-int ll_back_merge_fn(struct request_queue *q, struct request *req,
-                     struct bio *bio);
-int ll_front_merge_fn(struct request_queue *q, struct request *req, 
-                      struct bio *bio);
+void __blk_queue_split(struct request_queue *q, struct bio **bio,
+                unsigned int *nr_segs);
+int ll_back_merge_fn(struct request *req, struct bio *bio,
+                unsigned int nr_segs);
+int ll_front_merge_fn(struct request *req,  struct bio *bio,
+                unsigned int nr_segs);
 struct request *attempt_back_merge(struct request_queue *q, struct request *rq);
 struct request *attempt_front_merge(struct request_queue *q, struct request *rq);
 int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
                                 struct request *next);
-void blk_recalc_rq_segments(struct request *rq);
+unsigned int blk_recalc_rq_segments(struct request *rq);
 void blk_rq_set_mixed_merge(struct request *rq);
 bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
diff --git a/block/genhd.c b/block/genhd.c
index 24654e1d83e6..97887e59f3b2 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1281,7 +1281,6 @@ int disk_expand_part_tbl(struct gendisk *disk, int partno)
         struct disk_part_tbl *new_ptbl;
         int len = old_ptbl ? old_ptbl->len : 0;
         int i, target;
-        size_t size;
 
         /*
          * check for int overflow, since we can get here from blkpg_ioctl()
@@ -1298,8 +1297,8 @@ int disk_expand_part_tbl(struct gendisk *disk, int partno)
         if (target <= len)
                 return 0;
 
-        size = sizeof(*new_ptbl) + target * sizeof(new_ptbl->part[0]);
-        new_ptbl = kzalloc_node(size, GFP_KERNEL, disk->node_id);
+        new_ptbl = kzalloc_node(struct_size(new_ptbl, part, target), GFP_KERNEL,
+                                disk->node_id);
         if (!new_ptbl)
                 return -ENOMEM;
 
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index c3b05119cebd..34dcea0ef637 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -562,7 +562,8 @@ static void kyber_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
         }
 }
 
-static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
+static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio,
+                unsigned int nr_segs)
 {
         struct kyber_hctx_data *khd = hctx->sched_data;
         struct blk_mq_ctx *ctx = blk_mq_get_ctx(hctx->queue);
@@ -572,9 +573,8 @@ static bool kyber_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
         bool merged;
 
         spin_lock(&kcq->lock);
-        merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio);
+        merged = blk_mq_bio_list_merge(hctx->queue, rq_list, bio, nr_segs);
         spin_unlock(&kcq->lock);
-        blk_mq_put_ctx(ctx);
 
         return merged;
 }
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 1876f5712bfd..b8a682b5a1bb 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -469,7 +469,8 @@ static int dd_request_merge(struct request_queue *q, struct request **rq,
         return ELEVATOR_NO_MERGE;
 }
 
-static bool dd_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
+static bool dd_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio,
+                unsigned int nr_segs)
 {
         struct request_queue *q = hctx->queue;
         struct deadline_data *dd = q->elevator->elevator_data;
@@ -477,7 +478,7 @@ static bool dd_bio_merge(struct blk_mq_hw_ctx *hctx, struct bio *bio)
         bool ret;
 
         spin_lock(&dd->lock);
-        ret = blk_mq_sched_try_merge(q, bio, &free);
+        ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free);
         spin_unlock(&dd->lock);
 
         if (free)
diff --git a/block/opal_proto.h b/block/opal_proto.h
index d9a05ad02eb5..466ec7be16ef 100644
--- a/block/opal_proto.h
+++ b/block/opal_proto.h
@@ -98,6 +98,7 @@ enum opal_uid {
         OPAL_ENTERPRISE_BANDMASTER0_UID,
         OPAL_ENTERPRISE_ERASEMASTER_UID,
         /* tables */
+        OPAL_TABLE_TABLE,
         OPAL_LOCKINGRANGE_GLOBAL,
         OPAL_LOCKINGRANGE_ACE_RDLOCKED,
         OPAL_LOCKINGRANGE_ACE_WRLOCKED,
@@ -152,6 +153,21 @@ enum opal_token {
         OPAL_STARTCOLUMN = 0x03,
         OPAL_ENDCOLUMN = 0x04,
         OPAL_VALUES = 0x01,
+        /* table table */
+        OPAL_TABLE_UID = 0x00,
+        OPAL_TABLE_NAME = 0x01,
+        OPAL_TABLE_COMMON = 0x02,
+        OPAL_TABLE_TEMPLATE = 0x03,
+        OPAL_TABLE_KIND = 0x04,
+        OPAL_TABLE_COLUMN = 0x05,
+        OPAL_TABLE_COLUMNS = 0x06,
+        OPAL_TABLE_ROWS = 0x07,
+        OPAL_TABLE_ROWS_FREE = 0x08,
+        OPAL_TABLE_ROW_BYTES = 0x09,
+        OPAL_TABLE_LASTID = 0x0A,
+        OPAL_TABLE_MIN = 0x0B,
+        OPAL_TABLE_MAX = 0x0C,
+
         /* authority table */
         OPAL_PIN = 0x03,
         /* locking tokens */
diff --git a/block/sed-opal.c b/block/sed-opal.c
index a46e8d13e16d..7e1a444a25b2 100644
--- a/block/sed-opal.c
+++ b/block/sed-opal.c
@@ -26,6 +26,9 @@
 #define IO_BUFFER_LENGTH 2048
 #define MAX_TOKS 64
 
+/* Number of bytes needed by cmd_finalize. */
+#define CMD_FINALIZE_BYTES_NEEDED 7
+
 struct opal_step {
         int (*fn)(struct opal_dev *dev, void *data);
         void *data;
@@ -127,6 +130,8 @@ static const u8 opaluid[][OPAL_UID_LENGTH] = {
 
         /* tables */
 
+        [OPAL_TABLE_TABLE]
+                { 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01 },
         [OPAL_LOCKINGRANGE_GLOBAL] =
                 { 0x00, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x01 },
         [OPAL_LOCKINGRANGE_ACE_RDLOCKED] =
@@ -523,12 +528,17 @@ static int opal_discovery0_step(struct opal_dev *dev)
         return execute_step(dev, &discovery0_step, 0);
 }
 
+static size_t remaining_size(struct opal_dev *cmd)
+{
+        return IO_BUFFER_LENGTH - cmd->pos;
+}
+
 static bool can_add(int *err, struct opal_dev *cmd, size_t len)
 {
         if (*err)
                 return false;
 
-        if (len > IO_BUFFER_LENGTH || cmd->pos > IO_BUFFER_LENGTH - len) {
+        if (remaining_size(cmd) < len) {
                 pr_debug("Error adding %zu bytes: end of buffer.\n", len);
                 *err = -ERANGE;
                 return false;
@@ -674,7 +684,11 @@ static int cmd_finalize(struct opal_dev *cmd, u32 hsn, u32 tsn)
         struct opal_header *hdr;
         int err = 0;
 
-        /* close the parameter list opened from cmd_start */
+        /*
+         * Close the parameter list opened from cmd_start.
+         * The number of bytes added must be equal to
+         * CMD_FINALIZE_BYTES_NEEDED.
+         */
         add_token_u8(&err, cmd, OPAL_ENDLIST);
 
         add_token_u8(&err, cmd, OPAL_ENDOFDATA);
@@ -1119,6 +1133,29 @@ static int generic_get_column(struct opal_dev *dev, const u8 *table,
         return finalize_and_send(dev, parse_and_check_status);
 }
 
+/*
+ * see TCG SAS 5.3.2.3 for a description of the available columns
+ *
+ * the result is provided in dev->resp->tok[4]
+ */
+static int generic_get_table_info(struct opal_dev *dev, enum opal_uid table,
+                                  u64 column)
+{
+        u8 uid[OPAL_UID_LENGTH];
+        const unsigned int half = OPAL_UID_LENGTH/2;
+
+        /* sed-opal UIDs can be split in two halves:
+         *  first:  actual table index
+         *  second: relative index in the table
+         * so we have to get the first half of the OPAL_TABLE_TABLE and use the
+         * first part of the target table as relative index into that table
+         */
+        memcpy(uid, opaluid[OPAL_TABLE_TABLE], half);
+        memcpy(uid+half, opaluid[table], half);
+
+        return generic_get_column(dev, uid, column);
+}
+
 static int gen_key(struct opal_dev *dev, void *data)
 {
         u8 uid[OPAL_UID_LENGTH];
@@ -1307,6 +1344,7 @@ static int start_generic_opal_session(struct opal_dev *dev,
                 break;
         case OPAL_ADMIN1_UID:
         case OPAL_SID_UID:
+        case OPAL_PSID_UID:
                 add_token_u8(&err, dev, OPAL_STARTNAME);
                 add_token_u8(&err, dev, 0); /* HostChallenge */
                 add_token_bytestring(&err, dev, key, key_len);
@@ -1367,6 +1405,16 @@ static int start_admin1LSP_opal_session(struct opal_dev *dev, void *data)
                                           key->key, key->key_len);
 }
 
+static int start_PSID_opal_session(struct opal_dev *dev, void *data)
+{
+        const struct opal_key *okey = data;
+
+        return start_generic_opal_session(dev, OPAL_PSID_UID,
+                                          OPAL_ADMINSP_UID,
+                                          okey->key,
+                                          okey->key_len);
+}
+
 static int start_auth_opal_session(struct opal_dev *dev, void *data)
 {
         struct opal_session_info *session = data;
@@ -1525,6 +1573,72 @@ static int set_mbr_enable_disable(struct opal_dev *dev, void *data)
         return finalize_and_send(dev, parse_and_check_status);
 }
 
+static int write_shadow_mbr(struct opal_dev *dev, void *data)
+{
+        struct opal_shadow_mbr *shadow = data;
+        const u8 __user *src;
+        u8 *dst;
+        size_t off = 0;
+        u64 len;
+        int err = 0;
+
+        /* do we fit in the available shadow mbr space? */
+        err = generic_get_table_info(dev, OPAL_MBR, OPAL_TABLE_ROWS);
+        if (err) {
+                pr_debug("MBR: could not get shadow size\n");
+                return err;
+        }
+
+        len = response_get_u64(&dev->parsed, 4);
+        if (shadow->size > len || shadow->offset > len - shadow->size) {
+                pr_debug("MBR: does not fit in shadow (%llu vs. %llu)\n",
+                         shadow->offset + shadow->size, len);
+                return -ENOSPC;
+        }
+
+        /* do the actual transmission(s) */
+        src = (u8 __user *)(uintptr_t)shadow->data;
+        while (off < shadow->size) {
+                err = cmd_start(dev, opaluid[OPAL_MBR], opalmethod[OPAL_SET]);
+                add_token_u8(&err, dev, OPAL_STARTNAME);
+                add_token_u8(&err, dev, OPAL_WHERE);
+                add_token_u64(&err, dev, shadow->offset + off);
+                add_token_u8(&err, dev, OPAL_ENDNAME);
+
+                add_token_u8(&err, dev, OPAL_STARTNAME);
+                add_token_u8(&err, dev, OPAL_VALUES);
+
+                /*
+                 * The bytestring header is either 1 or 2 bytes, so assume 2.
+                 * There also needs to be enough space to accommodate the
+                 * trailing OPAL_ENDNAME (1 byte) and tokens added by
+                 * cmd_finalize.
+                 */
+                len = min(remaining_size(dev) - (2+1+CMD_FINALIZE_BYTES_NEEDED),
+                          (size_t)(shadow->size - off));
+                pr_debug("MBR: write bytes %zu+%llu/%llu\n",
+                         off, len, shadow->size);
+
+                dst = add_bytestring_header(&err, dev, len);
+                if (!dst)
+                        break;
+                if (copy_from_user(dst, src + off, len))
+                        err = -EFAULT;
+                dev->pos += len;
+
+                add_token_u8(&err, dev, OPAL_ENDNAME);
+                if (err)
+                        break;
+
+                err = finalize_and_send(dev, parse_and_check_status);
+                if (err)
+                        break;
+
+                off += len;
+        }
+        return err;
+}
+
 static int generic_pw_cmd(u8 *key, size_t key_len, u8 *cpin_uid,
                           struct opal_dev *dev)
 {
@@ -1978,6 +2092,50 @@ static int opal_enable_disable_shadow_mbr(struct opal_dev *dev,
         return ret;
 }
 
+static int opal_set_mbr_done(struct opal_dev *dev,
+                             struct opal_mbr_done *mbr_done)
+{
+        u8 mbr_done_tf = mbr_done->done_flag == OPAL_MBR_DONE ?
+                OPAL_TRUE : OPAL_FALSE;
+
+        const struct opal_step mbr_steps[] = {
+                { start_admin1LSP_opal_session, &mbr_done->key },
+                { set_mbr_done, &mbr_done_tf },
+                { end_opal_session, }
+        };
+        int ret;
+
+        if (mbr_done->done_flag != OPAL_MBR_DONE &&
+            mbr_done->done_flag != OPAL_MBR_NOT_DONE)
+                return -EINVAL;
+
+        mutex_lock(&dev->dev_lock);
+        setup_opal_dev(dev);
+        ret = execute_steps(dev, mbr_steps, ARRAY_SIZE(mbr_steps));
+        mutex_unlock(&dev->dev_lock);
+        return ret;
+}
+
+static int opal_write_shadow_mbr(struct opal_dev *dev,
+                                 struct opal_shadow_mbr *info)
+{
+        const struct opal_step mbr_steps[] = {
+                { start_admin1LSP_opal_session, &info->key },
+                { write_shadow_mbr, info },
+                { end_opal_session, }
+        };
+        int ret;
+
+        if (info->size == 0)
+                return 0;
+
+        mutex_lock(&dev->dev_lock);
+        setup_opal_dev(dev);
+        ret = execute_steps(dev, mbr_steps, ARRAY_SIZE(mbr_steps));
+        mutex_unlock(&dev->dev_lock);
+        return ret;
+}
+
 static int opal_save(struct opal_dev *dev, struct opal_lock_unlock *lk_unlk)
 {
         struct opal_suspend_data *suspend;
@@ -2030,17 +2188,28 @@ static int opal_add_user_to_lr(struct opal_dev *dev,
         return ret;
 }
 
-static int opal_reverttper(struct opal_dev *dev, struct opal_key *opal)
+static int opal_reverttper(struct opal_dev *dev, struct opal_key *opal, bool psid)
 {
+        /* controller will terminate session */
         const struct opal_step revert_steps[] = {
                 { start_SIDASP_opal_session, opal },
-                { revert_tper, } /* controller will terminate session */
+                { revert_tper, }
+        };
+        const struct opal_step psid_revert_steps[] = {
+                { start_PSID_opal_session, opal },
+                { revert_tper, }
         };
+
         int ret;
 
         mutex_lock(&dev->dev_lock);
         setup_opal_dev(dev);
-        ret = execute_steps(dev, revert_steps, ARRAY_SIZE(revert_steps));
+        if (psid)
+                ret = execute_steps(dev, psid_revert_steps,
+                                    ARRAY_SIZE(psid_revert_steps));
+        else
+                ret = execute_steps(dev, revert_steps,
+                                    ARRAY_SIZE(revert_steps));
         mutex_unlock(&dev->dev_lock);
 
         /*
@@ -2092,8 +2261,7 @@ static int opal_lock_unlock(struct opal_dev *dev,
 {
         int ret;
 
-        if (lk_unlk->session.who < OPAL_ADMIN1 ||
-            lk_unlk->session.who > OPAL_USER9)
+        if (lk_unlk->session.who > OPAL_USER9)
                 return -EINVAL;
 
         mutex_lock(&dev->dev_lock);
@@ -2171,9 +2339,7 @@ static int opal_set_new_pw(struct opal_dev *dev, struct opal_new_pw *opal_pw)
         };
         int ret;
 
-        if (opal_pw->session.who < OPAL_ADMIN1 ||
-            opal_pw->session.who > OPAL_USER9  ||
-            opal_pw->new_user_pw.who < OPAL_ADMIN1 ||
+        if (opal_pw->session.who > OPAL_USER9  ||
             opal_pw->new_user_pw.who > OPAL_USER9)
                 return -EINVAL;
 
@@ -2280,7 +2446,7 @@ int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg)
                 ret = opal_activate_user(dev, p);
                 break;
         case IOC_OPAL_REVERT_TPR:
-                ret = opal_reverttper(dev, p);
+                ret = opal_reverttper(dev, p, false);
                 break;
         case IOC_OPAL_LR_SETUP:
                 ret = opal_setup_locking_range(dev, p);
@@ -2291,12 +2457,21 @@ int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg)
         case IOC_OPAL_ENABLE_DISABLE_MBR:
                 ret = opal_enable_disable_shadow_mbr(dev, p);
                 break;
+        case IOC_OPAL_MBR_DONE:
+                ret = opal_set_mbr_done(dev, p);
+                break;
+        case IOC_OPAL_WRITE_SHADOW_MBR:
+                ret = opal_write_shadow_mbr(dev, p);
+                break;
         case IOC_OPAL_ERASE_LR:
                 ret = opal_erase_locking_range(dev, p);
                 break;
         case IOC_OPAL_SECURE_ERASE_LR:
                 ret = opal_secure_erase_locking_range(dev, p);
                 break;
+        case IOC_OPAL_PSID_REVERT_TPR:
+                ret = opal_reverttper(dev, p, true);
+                break;
         default:
                 break;
         }