Merge branch 'for-3.16/core' of git://git.kernel.dk/linux-block into next

Pull block core updates from Jens Axboe:
 "It's a big(ish) round this time, lots of development effort has gone
  into blk-mq in the last 3 months.  Generally we're heading to where
  3.16 will be a feature complete and performant blk-mq.  scsi-mq is
  progressing nicely and will hopefully be in 3.17.  A nvme port is in
  progress, and the Micron pci-e flash driver, mtip32xx, is converted
  and will be sent in with the driver pull request for 3.16.

  This pull request contains:

   - Lots of prep and support patches for scsi-mq have been integrated.
     All from Christoph.

   - API and code cleanups for blk-mq from Christoph.

   - Lots of good corner case and error handling cleanup fixes for
     blk-mq from Ming Lei.

   - A flew of blk-mq updates from me:

     * Provide strict mappings so that the driver can rely on the CPU
       to queue mapping.  This enables optimizations in the driver.

     * Provided a bitmap tagging instead of percpu_ida, which never
       really worked well for blk-mq.  percpu_ida relies on the fact
       that we have a lot more tags available than we really need, it
       fails miserably for cases where we exhaust (or are close to
       exhausting) the tag space.

     * Provide sane support for shared tag maps, as utilized by scsi-mq

     * Various fixes for IO timeouts.

     * API cleanups, and lots of perf tweaks and optimizations.

   - Remove 'buffer' from struct request.  This is ancient code, from
     when requests were always virtually mapped.  Kill it, to reclaim
     some space in struct request.  From me.

   - Remove 'magic' from blk_plug.  Since we store these on the stack
     and since we've never caught any actual bugs with this, lets just
     get rid of it.  From me.

   - Only call part_in_flight() once for IO completion, as includes two
     atomic reads.  Hopefully we'll get a better implementation soon, as
     the part IO stats are now one of the more expensive parts of doing
     IO on blk-mq.  From me.

   - File migration of block code from {mm,fs}/ to block/.  This
     includes bio.c, bio-integrity.c, bounce.c, and ioprio.c.  From me,
     from a discussion on lkml.

  That should describe the meat of the pull request.  Also has various
  little fixes and cleanups from Dave Jones, Shaohua Li, Duan Jiong,
  Fengguang Wu, Fabian Frederick, Randy Dunlap, Robert Elliott, and Sam
  Bradshaw"

* 'for-3.16/core' of git://git.kernel.dk/linux-block: (100 commits)
  blk-mq: push IPI or local end_io decision to __blk_mq_complete_request()
  blk-mq: remember to start timeout handler for direct queue
  block: ensure that the timer is always added
  blk-mq: blk_mq_unregister_hctx() can be static
  blk-mq: make the sysfs mq/ layout reflect current mappings
  blk-mq: blk_mq_tag_to_rq should handle flush request
  block: remove dead code in scsi_ioctl:blk_verify_command
  blk-mq: request initialization optimizations
  block: add queue flag for disabling SG merging
  block: remove 'magic' from struct blk_plug
  blk-mq: remove alloc_hctx and free_hctx methods
  blk-mq: add file comments and update copyright notices
  blk-mq: remove blk_mq_alloc_request_pinned
  blk-mq: do not use blk_mq_alloc_request_pinned in blk_mq_map_request
  blk-mq: remove blk_mq_wait_for_tags
  blk-mq: initialize request in __blk_mq_alloc_request
  blk-mq: merge blk_mq_alloc_reserved_request into blk_mq_alloc_request
  blk-mq: add helper to insert requests from irq context
  blk-mq: remove stale comment for blk_mq_complete_request()
  blk-mq: allow non-softirq completions
  ...

4 files changed, 82 insertions, 50 deletions

diff --git a/include/linux/bio.h b/include/linux/bio.h
index bba550826921..5a645769f020 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -333,7 +333,7 @@ static inline struct bio *bio_next_split(struct bio *bio, int sectors,
 
 extern struct bio_set *bioset_create(unsigned int, unsigned int);
 extern void bioset_free(struct bio_set *);
-extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries);
+extern mempool_t *biovec_create_pool(int pool_entries);
 
 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
 extern void bio_put(struct bio *);
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 0120451545d8..c15128833100 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -8,7 +8,13 @@ struct blk_mq_tags;
 struct blk_mq_cpu_notifier {
         struct list_head list;
         void *data;
-        void (*notify)(void *data, unsigned long action, unsigned int cpu);
+        int (*notify)(void *data, unsigned long action, unsigned int cpu);
+};
+
+struct blk_mq_ctxmap {
+        unsigned int map_size;
+        unsigned int bits_per_word;
+        struct blk_align_bitmap *map;
 };
 
 struct blk_mq_hw_ctx {
@@ -18,7 +24,11 @@ struct blk_mq_hw_ctx {
         } ____cacheline_aligned_in_smp;
 
         unsigned long           state;          /* BLK_MQ_S_* flags */
-        struct delayed_work     delayed_work;
+        struct delayed_work     run_work;
+        struct delayed_work     delay_work;
+        cpumask_var_t           cpumask;
+        int                     next_cpu;
+        int                     next_cpu_batch;
 
         unsigned long           flags;          /* BLK_MQ_F_* flags */
 
@@ -27,13 +37,13 @@ struct blk_mq_hw_ctx {
 
         void                    *driver_data;
 
+        struct blk_mq_ctxmap    ctx_map;
+
         unsigned int            nr_ctx;
         struct blk_mq_ctx       **ctxs;
-        unsigned int            nr_ctx_map;
-        unsigned long           *ctx_map;
 
-        struct request          **rqs;
-        struct list_head        page_list;
+        unsigned int            wait_index;
+
         struct blk_mq_tags      *tags;
 
         unsigned long           queued;
@@ -41,31 +51,40 @@ struct blk_mq_hw_ctx {
 #define BLK_MQ_MAX_DISPATCH_ORDER       10
         unsigned long           dispatched[BLK_MQ_MAX_DISPATCH_ORDER];
 
-        unsigned int            queue_depth;
         unsigned int            numa_node;
         unsigned int            cmd_size;       /* per-request extra data */
 
+        atomic_t                nr_active;
+
         struct blk_mq_cpu_notifier      cpu_notifier;
         struct kobject          kobj;
 };
 
-struct blk_mq_reg {
+struct blk_mq_tag_set {
         struct blk_mq_ops       *ops;
         unsigned int            nr_hw_queues;
-        unsigned int            queue_depth;
+        unsigned int            queue_depth;    /* max hw supported */
         unsigned int            reserved_tags;
         unsigned int            cmd_size;       /* per-request extra data */
         int                     numa_node;
         unsigned int            timeout;
         unsigned int            flags;          /* BLK_MQ_F_* */
+        void                    *driver_data;
+
+        struct blk_mq_tags      **tags;
+
+        struct mutex            tag_list_lock;
+        struct list_head        tag_list;
 };
 
 typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, struct request *);
 typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int);
-typedef struct blk_mq_hw_ctx *(alloc_hctx_fn)(struct blk_mq_reg *,unsigned int);
-typedef void (free_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);
 typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int);
 typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);
+typedef int (init_request_fn)(void *, struct request *, unsigned int,
+                unsigned int, unsigned int);
+typedef void (exit_request_fn)(void *, struct request *, unsigned int,
+                unsigned int);
 
 struct blk_mq_ops {
         /*
@@ -86,18 +105,20 @@ struct blk_mq_ops {
         softirq_done_fn         *complete;
 
         /*
-         * Override for hctx allocations (should probably go)
-         */
-        alloc_hctx_fn           *alloc_hctx;
-        free_hctx_fn            *free_hctx;
-
-        /*
          * Called when the block layer side of a hardware queue has been
          * set up, allowing the driver to allocate/init matching structures.
          * Ditto for exit/teardown.
          */
         init_hctx_fn            *init_hctx;
         exit_hctx_fn            *exit_hctx;
+
+        /*
+         * Called for every command allocated by the block layer to allow
+         * the driver to set up driver specific data.
+         * Ditto for exit/teardown.
+         */
+        init_request_fn         *init_request;
+        exit_request_fn         *exit_request;
 };
 
 enum {
@@ -107,18 +128,24 @@ enum {
 
         BLK_MQ_F_SHOULD_MERGE   = 1 << 0,
         BLK_MQ_F_SHOULD_SORT    = 1 << 1,
-        BLK_MQ_F_SHOULD_IPI     = 1 << 2,
+        BLK_MQ_F_TAG_SHARED     = 1 << 2,
+        BLK_MQ_F_SG_MERGE       = 1 << 3,
+        BLK_MQ_F_SYSFS_UP       = 1 << 4,
 
         BLK_MQ_S_STOPPED        = 0,
+        BLK_MQ_S_TAG_ACTIVE     = 1,
 
         BLK_MQ_MAX_DEPTH        = 2048,
+
+        BLK_MQ_CPU_WORK_BATCH   = 8,
 };
 
-struct request_queue *blk_mq_init_queue(struct blk_mq_reg *, void *);
+struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
 int blk_mq_register_disk(struct gendisk *);
 void blk_mq_unregister_disk(struct gendisk *);
-int blk_mq_init_commands(struct request_queue *, int (*init)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data);
-void blk_mq_free_commands(struct request_queue *, void (*free)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data);
+
+int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set);
+void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
 
 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
 
@@ -126,28 +153,28 @@ void blk_mq_insert_request(struct request *, bool, bool, bool);
 void blk_mq_run_queues(struct request_queue *q, bool async);
 void blk_mq_free_request(struct request *rq);
 bool blk_mq_can_queue(struct blk_mq_hw_ctx *);
-struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp);
-struct request *blk_mq_alloc_reserved_request(struct request_queue *q, int rw, gfp_t gfp);
-struct request *blk_mq_rq_from_tag(struct request_queue *q, unsigned int tag);
+struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
+                gfp_t gfp, bool reserved);
+struct request *blk_mq_tag_to_rq(struct blk_mq_hw_ctx *hctx, unsigned int tag);
 
 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index);
-struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_reg *, unsigned int);
-void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *, unsigned int);
+struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int);
 
-bool blk_mq_end_io_partial(struct request *rq, int error,
-                unsigned int nr_bytes);
-static inline void blk_mq_end_io(struct request *rq, int error)
-{
-        bool done = !blk_mq_end_io_partial(rq, error, blk_rq_bytes(rq));
-        BUG_ON(!done);
-}
+void blk_mq_end_io(struct request *rq, int error);
+void __blk_mq_end_io(struct request *rq, int error);
 
+void blk_mq_requeue_request(struct request *rq);
+void blk_mq_add_to_requeue_list(struct request *rq, bool at_head);
+void blk_mq_kick_requeue_list(struct request_queue *q);
 void blk_mq_complete_request(struct request *rq);
 
 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_stop_hw_queues(struct request_queue *q);
-void blk_mq_start_stopped_hw_queues(struct request_queue *q);
+void blk_mq_start_hw_queues(struct request_queue *q);
+void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
+void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
+void blk_mq_tag_busy_iter(struct blk_mq_tags *tags, void (*fn)(void *data, unsigned long *), void *data);
 
 /*
  * Driver command data is immediately after the request. So subtract request
@@ -162,12 +189,6 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq)
         return (void *) rq + sizeof(*rq);
 }
 
-static inline struct request *blk_mq_tag_to_rq(struct blk_mq_hw_ctx *hctx,
-                                               unsigned int tag)
-{
-        return hctx->rqs[tag];
-}
-
 #define queue_for_each_hw_ctx(q, hctx, i)                               \
         for ((i) = 0; (i) < (q)->nr_hw_queues &&                        \
              ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++)
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index aa0eaa2d0bd8..d8e4cea23a25 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -190,6 +190,7 @@ enum rq_flag_bits {
         __REQ_PM,               /* runtime pm request */
         __REQ_END,              /* last of chain of requests */
         __REQ_HASHED,           /* on IO scheduler merge hash */
+        __REQ_MQ_INFLIGHT,      /* track inflight for MQ */
         __REQ_NR_BITS,          /* stops here */
 };
 
@@ -243,5 +244,6 @@ enum rq_flag_bits {
 #define REQ_PM                  (1ULL << __REQ_PM)
 #define REQ_END                 (1ULL << __REQ_END)
 #define REQ_HASHED              (1ULL << __REQ_HASHED)
+#define REQ_MQ_INFLIGHT         (1ULL << __REQ_MQ_INFLIGHT)
 
 #endif /* __LINUX_BLK_TYPES_H */
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 0d84981ee03f..695b9fd41efe 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -90,15 +90,15 @@ enum rq_cmd_type_bits {
 #define BLK_MAX_CDB     16
 
 /*
- * try to put the fields that are referenced together in the same cacheline.
- * if you modify this structure, be sure to check block/blk-core.c:blk_rq_init()
- * as well!
+ * Try to put the fields that are referenced together in the same cacheline.
+ *
+ * If you modify this structure, make sure to update blk_rq_init() and
+ * especially blk_mq_rq_ctx_init() to take care of the added fields.
  */
 struct request {
         struct list_head queuelist;
         union {
                 struct call_single_data csd;
-                struct work_struct mq_flush_work;
                 unsigned long fifo_time;
         };
 
@@ -178,7 +178,6 @@ struct request {
         unsigned short ioprio;
 
         void *special;          /* opaque pointer available for LLD use */
-        char *buffer;           /* kaddr of the current segment if available */
 
         int tag;
         int errors;
@@ -463,6 +462,10 @@ struct request_queue {
         struct request          *flush_rq;
         spinlock_t              mq_flush_lock;
 
+        struct list_head        requeue_list;
+        spinlock_t              requeue_lock;
+        struct work_struct      requeue_work;
+
         struct mutex            sysfs_lock;
 
         int                     bypass_depth;
@@ -481,6 +484,9 @@ struct request_queue {
         wait_queue_head_t       mq_freeze_wq;
         struct percpu_counter   mq_usage_counter;
         struct list_head        all_q_node;
+
+        struct blk_mq_tag_set   *tag_set;
+        struct list_head        tag_set_list;
 };
 
 #define QUEUE_FLAG_QUEUED       1       /* uses generic tag queueing */
@@ -504,6 +510,7 @@ struct request_queue {
 #define QUEUE_FLAG_SAME_FORCE  18       /* force complete on same CPU */
 #define QUEUE_FLAG_DEAD        19       /* queue tear-down finished */
 #define QUEUE_FLAG_INIT_DONE   20       /* queue is initialized */
+#define QUEUE_FLAG_NO_SG_MERGE 21       /* don't attempt to merge SG segments*/
 
 #define QUEUE_FLAG_DEFAULT      ((1 << QUEUE_FLAG_IO_STAT) |            \
                                  (1 << QUEUE_FLAG_STACKABLE)    |       \
@@ -937,6 +944,7 @@ extern struct request *blk_fetch_request(struct request_queue *q);
  */
 extern bool blk_update_request(struct request *rq, int error,
                                unsigned int nr_bytes);
+extern void blk_finish_request(struct request *rq, int error);
 extern bool blk_end_request(struct request *rq, int error,
                             unsigned int nr_bytes);
 extern void blk_end_request_all(struct request *rq, int error);
@@ -1053,7 +1061,6 @@ static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
  * schedule() where blk_schedule_flush_plug() is called.
  */
 struct blk_plug {
-        unsigned long magic; /* detect uninitialized use-cases */
         struct list_head list; /* requests */
         struct list_head mq_list; /* blk-mq requests */
         struct list_head cb_list; /* md requires an unplug callback */
@@ -1102,7 +1109,8 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk)
 /*
  * tag stuff
  */
-#define blk_rq_tagged(rq)               ((rq)->cmd_flags & REQ_QUEUED)
+#define blk_rq_tagged(rq) \
+        ((rq)->mq_ctx || ((rq)->cmd_flags & REQ_QUEUED))
 extern int blk_queue_start_tag(struct request_queue *, struct request *);
 extern struct request *blk_queue_find_tag(struct request_queue *, int);
 extern void blk_queue_end_tag(struct request_queue *, struct request *);
@@ -1370,8 +1378,9 @@ static inline void put_dev_sector(Sector p)
 }
 
 struct work_struct;
-int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
-int kblockd_schedule_delayed_work(struct request_queue *q, struct delayed_work *dwork, unsigned long delay);
+int kblockd_schedule_work(struct work_struct *work);
+int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
+int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
 
 #ifdef CONFIG_BLK_CGROUP
 /*