From 75ad23bc0fcb4f992a5d06982bf0857ab1738e9e Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Tue, 29 Apr 2008 14:48:33 +0200 Subject: block: make queue flags non-atomic We can save some atomic ops in the IO path, if we clearly define the rules of how to modify the queue flags. Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c5065e3d2ca9..8ca481cd7d73 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -408,6 +408,30 @@ struct request_queue #define QUEUE_FLAG_ELVSWITCH 8 /* don't use elevator, just do FIFO */ #define QUEUE_FLAG_BIDI 9 /* queue supports bidi requests */ +static inline void queue_flag_set_unlocked(unsigned int flag, + struct request_queue *q) +{ + __set_bit(flag, &q->queue_flags); +} + +static inline void queue_flag_set(unsigned int flag, struct request_queue *q) +{ + WARN_ON_ONCE(!spin_is_locked(q->queue_lock)); + __set_bit(flag, &q->queue_flags); +} + +static inline void queue_flag_clear_unlocked(unsigned int flag, + struct request_queue *q) +{ + __clear_bit(flag, &q->queue_flags); +} + +static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) +{ + WARN_ON_ONCE(!spin_is_locked(q->queue_lock)); + __clear_bit(flag, &q->queue_flags); +} + enum { /* * Hardbarrier is supported with one of the following methods. @@ -496,17 +520,17 @@ static inline int blk_queue_full(struct request_queue *q, int rw) static inline void blk_set_queue_full(struct request_queue *q, int rw) { if (rw == READ) - set_bit(QUEUE_FLAG_READFULL, &q->queue_flags); + queue_flag_set(QUEUE_FLAG_READFULL, q); else - set_bit(QUEUE_FLAG_WRITEFULL, &q->queue_flags); + queue_flag_set(QUEUE_FLAG_WRITEFULL, q); } static inline void blk_clear_queue_full(struct request_queue *q, int rw) { if (rw == READ) - clear_bit(QUEUE_FLAG_READFULL, &q->queue_flags); + queue_flag_clear(QUEUE_FLAG_READFULL, q); else - clear_bit(QUEUE_FLAG_WRITEFULL, &q->queue_flags); + queue_flag_clear(QUEUE_FLAG_WRITEFULL, q); } @@ -626,6 +650,7 @@ extern void blk_start_queue(struct request_queue *q); extern void blk_stop_queue(struct request_queue *q); extern void blk_sync_queue(struct request_queue *q); extern void __blk_stop_queue(struct request_queue *q); +extern void __blk_run_queue(struct request_queue *); extern void blk_run_queue(struct request_queue *); extern void blk_start_queueing(struct request_queue *); extern int blk_rq_map_user(struct request_queue *, struct request *, void __user *, unsigned long); -- cgit v1.2.2 From 2a4aa30c5f967eb6ae874c67fa6fceeee84815f9 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 29 Apr 2008 09:54:36 +0200 Subject: block: rename and export rq_init() This rename rq_init() blk_rq_init() and export it. Any path that hands the request to the block layer needs to call it to initialize the request. This is a preparation for large command support, which needs to initialize the request in a proper way (that is, just doing a memset() will not work). Signed-off-by: FUJITA Tomonori Cc: Jens Axboe Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 8ca481cd7d73..d17032c347c0 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -607,6 +607,7 @@ extern int blk_register_queue(struct gendisk *disk); extern void blk_unregister_queue(struct gendisk *disk); extern void register_disk(struct gendisk *dev); extern void generic_make_request(struct bio *bio); +extern void blk_rq_init(struct request_queue *q, struct request *rq); extern void blk_put_request(struct request *); extern void __blk_put_request(struct request_queue *, struct request *); extern void blk_end_sync_rq(struct request *rq, int error); -- cgit v1.2.2 From d7e3c3249ef23b4617393c69fe464765b4ff1645 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 29 Apr 2008 09:54:39 +0200 Subject: block: add large command support This patch changes rq->cmd from the static array to a pointer to support large commands. We rarely handle large commands. So for optimization, a struct request still has a static array for a command. rq_init sets rq->cmd pointer to the static array. Signed-off-by: FUJITA Tomonori Cc: Jens Axboe Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d17032c347c0..08df1ea8bac4 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -215,8 +215,9 @@ struct request { /* * when request is used as a packet command carrier */ - unsigned int cmd_len; - unsigned char cmd[BLK_MAX_CDB]; + unsigned short cmd_len; + unsigned char __cmd[BLK_MAX_CDB]; + unsigned char *cmd; unsigned int data_len; unsigned int extra_len; /* length of alignment and padding */ -- cgit v1.2.2 From ac9fafa1243640349aa481adf473db283a695766 Mon Sep 17 00:00:00 2001 From: "Alan D. Brunelle" Date: Tue, 29 Apr 2008 14:44:19 +0200 Subject: block: Skip I/O merges when disabled The block I/O + elevator + I/O scheduler code spend a lot of time trying to merge I/Os -- rightfully so under "normal" circumstances. However, if one were to know that the incoming I/O stream was /very/ random in nature, the cycles are wasted. This patch adds a per-request_queue tunable that (when set) disables merge attempts (beyond the simple one-hit cache check), thus freeing up a non-trivial amount of CPU cycles. Signed-off-by: Alan D. Brunelle Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 08df1ea8bac4..c09696a90d6a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -408,6 +408,7 @@ struct request_queue #define QUEUE_FLAG_PLUGGED 7 /* queue is plugged */ #define QUEUE_FLAG_ELVSWITCH 8 /* don't use elevator, just do FIFO */ #define QUEUE_FLAG_BIDI 9 /* queue supports bidi requests */ +#define QUEUE_FLAG_NOMERGES 10 /* disable merge attempts */ static inline void queue_flag_set_unlocked(unsigned int flag, struct request_queue *q) @@ -476,6 +477,7 @@ enum { #define blk_queue_plugged(q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags) #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) +#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) #define blk_queue_flushing(q) ((q)->ordseq) #define blk_fs_request(rq) ((rq)->cmd_type == REQ_TYPE_FS) -- cgit v1.2.2 From 8f45c1a58a25c3a1a2f42521445e1e786c4c0b92 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Tue, 29 Apr 2008 10:16:38 -0700 Subject: block: fix queue locking verification The new queue_flag_set/clear() functions verify that the queue is locked, but in doing so they will actually instead oops if the queue lock hasn't been initialized at all. So fix the lock debug test to consider the "no lock" case to be unlocked. This way you get a nice WARN_ON_ONCE() instead of a fatal oops. Bug introduced by commit 75ad23bc0fcb4f992a5d06982bf0857ab1738e9e ("block: make queue flags non-atomic"). Cc: Jens Axboe Cc: Nick Piggin Signed-off-by: Linus Torvalds --- include/linux/blkdev.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index c09696a90d6a..95864b3ff298 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -410,6 +410,12 @@ struct request_queue #define QUEUE_FLAG_BIDI 9 /* queue supports bidi requests */ #define QUEUE_FLAG_NOMERGES 10 /* disable merge attempts */ +static inline int queue_is_locked(struct request_queue *q) +{ + spinlock_t *lock = q->queue_lock; + return lock && spin_is_locked(lock); +} + static inline void queue_flag_set_unlocked(unsigned int flag, struct request_queue *q) { @@ -418,7 +424,7 @@ static inline void queue_flag_set_unlocked(unsigned int flag, static inline void queue_flag_set(unsigned int flag, struct request_queue *q) { - WARN_ON_ONCE(!spin_is_locked(q->queue_lock)); + WARN_ON_ONCE(!queue_is_locked(q)); __set_bit(flag, &q->queue_flags); } @@ -430,7 +436,7 @@ static inline void queue_flag_clear_unlocked(unsigned int flag, static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) { - WARN_ON_ONCE(!spin_is_locked(q->queue_lock)); + WARN_ON_ONCE(!queue_is_locked(q)); __clear_bit(flag, &q->queue_flags); } -- cgit v1.2.2 From 7663c1e2792a9662b23dec6e19bfcd3d55360b8f Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 29 Apr 2008 21:31:27 +0200 Subject: Improve queue_is_locked() spin_is_locked() doesn't work on UP without spinlock debugging. Make it safer and just return 1 on UP, so we don't get false positives. The plan is to kill this debug function during the -rc cycle. Signed-off-by: Jens Axboe Signed-off-by: Linus Torvalds --- include/linux/blkdev.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 95864b3ff298..d2a1b71e93c3 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -412,8 +412,12 @@ struct request_queue static inline int queue_is_locked(struct request_queue *q) { +#ifdef CONFIG_SMP spinlock_t *lock = q->queue_lock; return lock && spin_is_locked(lock); +#else + return 1; +#endif } static inline void queue_flag_set_unlocked(unsigned int flag, -- cgit v1.2.2