1 files changed, 51 insertions, 25 deletions

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index ba43f408baa3..4a2ab7c85393 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -46,16 +46,23 @@ struct blkcg_gq;
 struct request;
 typedef void (rq_end_io_fn)(struct request *, int);
 
+#define BLK_RL_SYNCFULL         (1U << 0)
+#define BLK_RL_ASYNCFULL        (1U << 1)
+
 struct request_list {
+        struct request_queue    *q;     /* the queue this rl belongs to */
+#ifdef CONFIG_BLK_CGROUP
+        struct blkcg_gq         *blkg;  /* blkg this request pool belongs to */
+#endif
         /*
          * count[], starved[], and wait[] are indexed by
          * BLK_RW_SYNC/BLK_RW_ASYNC
          */
-        int count[2];
-        int starved[2];
-        int elvpriv;
-        mempool_t *rq_pool;
-        wait_queue_head_t wait[2];
+        int                     count[2];
+        int                     starved[2];
+        mempool_t               *rq_pool;
+        wait_queue_head_t       wait[2];
+        unsigned int            flags;
 };
 
 /*
@@ -138,6 +145,7 @@ struct request {
         struct hd_struct *part;
         unsigned long start_time;
 #ifdef CONFIG_BLK_CGROUP
+        struct request_list *rl;                /* rl this rq is alloced from */
         unsigned long long start_time_ns;
         unsigned long long io_start_time_ns;    /* when passed to hardware */
 #endif
@@ -282,11 +290,16 @@ struct request_queue {
         struct list_head        queue_head;
         struct request          *last_merge;
         struct elevator_queue   *elevator;
+        int                     nr_rqs[2];      /* # allocated [a]sync rqs */
+        int                     nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */
 
         /*
-         * the queue request freelist, one for reads and one for writes
+         * If blkcg is not used, @q->root_rl serves all requests.  If blkcg
+         * is used, root blkg allocates from @q->root_rl and all other
+         * blkgs from their own blkg->rl.  Which one to use should be
+         * determined using bio_request_list().
          */
-        struct request_list     rq;
+        struct request_list     root_rl;
 
         request_fn_proc         *request_fn;
         make_request_fn         *make_request_fn;
@@ -561,27 +574,25 @@ static inline bool rq_is_sync(struct request *rq)
         return rw_is_sync(rq->cmd_flags);
 }
 
-static inline int blk_queue_full(struct request_queue *q, int sync)
+static inline bool blk_rl_full(struct request_list *rl, bool sync)
 {
-        if (sync)
-                return test_bit(QUEUE_FLAG_SYNCFULL, &q->queue_flags);
-        return test_bit(QUEUE_FLAG_ASYNCFULL, &q->queue_flags);
+        unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
+
+        return rl->flags & flag;
 }
 
-static inline void blk_set_queue_full(struct request_queue *q, int sync)
+static inline void blk_set_rl_full(struct request_list *rl, bool sync)
 {
-        if (sync)
-                queue_flag_set(QUEUE_FLAG_SYNCFULL, q);
-        else
-                queue_flag_set(QUEUE_FLAG_ASYNCFULL, q);
+        unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
+
+        rl->flags |= flag;
 }
 
-static inline void blk_clear_queue_full(struct request_queue *q, int sync)
+static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
 {
-        if (sync)
-                queue_flag_clear(QUEUE_FLAG_SYNCFULL, q);
-        else
-                queue_flag_clear(QUEUE_FLAG_ASYNCFULL, q);
+        unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
+
+        rl->flags &= ~flag;
 }
 
 
@@ -590,7 +601,7 @@ static inline void blk_clear_queue_full(struct request_queue *q, int sync)
  * it already be started by driver.
  */
 #define RQ_NOMERGE_FLAGS        \
-        (REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_FLUSH | REQ_FUA)
+        (REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_FLUSH | REQ_FUA | REQ_DISCARD)
 #define rq_mergeable(rq)        \
         (!((rq)->cmd_flags & RQ_NOMERGE_FLAGS) && \
          (((rq)->cmd_flags & REQ_DISCARD) || \
@@ -827,7 +838,6 @@ extern bool __blk_end_request_err(struct request *rq, int error);
 extern void blk_complete_request(struct request *);
 extern void __blk_complete_request(struct request *);
 extern void blk_abort_request(struct request *);
-extern void blk_abort_queue(struct request_queue *);
 extern void blk_unprep_request(struct request *);
 
 /*
@@ -884,6 +894,8 @@ extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
 extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
 
 extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
+extern int blk_bio_map_sg(struct request_queue *q, struct bio *bio,
+                          struct scatterlist *sglist);
 extern void blk_dump_rq_flags(struct request *, char *);
 extern long nr_blockdev_pages(void);
 
@@ -912,11 +924,15 @@ struct blk_plug {
 };
 #define BLK_MAX_REQUEST_COUNT 16
 
+struct blk_plug_cb;
+typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
 struct blk_plug_cb {
         struct list_head list;
-        void (*callback)(struct blk_plug_cb *);
+        blk_plug_cb_fn callback;
+        void *data;
 };
-
+extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
+                                             void *data, int size);
 extern void blk_start_plug(struct blk_plug *);
 extern void blk_finish_plug(struct blk_plug *);
 extern void blk_flush_plug_list(struct blk_plug *, bool);
@@ -1125,6 +1141,16 @@ static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector
                 & (lim->discard_granularity - 1);
 }
 
+static inline int bdev_discard_alignment(struct block_device *bdev)
+{
+        struct request_queue *q = bdev_get_queue(bdev);
+
+        if (bdev != bdev->bd_contains)
+                return bdev->bd_part->discard_alignment;
+
+        return q->limits.discard_alignment;
+}
+
 static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
 {
         if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1)