diff options
author | Jens Axboe <axboe@fb.com> | 2015-01-23 16:18:49 -0500 |
---|---|---|
committer | Jens Axboe <axboe@fb.com> | 2015-01-23 16:18:49 -0500 |
commit | a4a1cc16a72fd957b4cc2bda12479117f5bec8f8 (patch) | |
tree | ae43584c8b2d1f244653691a606071f2524bd6ee | |
parent | 121c7ad4ef3f3a9e697c4b8b78d74e4d6847d9e4 (diff) | |
parent | 24391c0dc57c3756a219defaa781e68637d6ab7d (diff) |
Merge branch 'for-3.20/core' into for-3.20/drivers
We need the tagging changes for the libata conversion.
-rw-r--r-- | block/blk-lib.c | 30 | ||||
-rw-r--r-- | block/blk-mq-tag.c | 80 | ||||
-rw-r--r-- | block/blk-mq-tag.h | 4 | ||||
-rw-r--r-- | block/blk-mq.c | 3 | ||||
-rw-r--r-- | block/blk-tag.c | 33 | ||||
-rw-r--r-- | block/cfq-iosched.c | 9 | ||||
-rw-r--r-- | block/ioctl.c | 2 | ||||
-rw-r--r-- | block/partitions/check.c | 12 | ||||
-rw-r--r-- | drivers/block/drbd/drbd_receiver.c | 2 | ||||
-rw-r--r-- | drivers/block/osdblk.c | 2 | ||||
-rw-r--r-- | drivers/scsi/scsi_lib.c | 2 | ||||
-rw-r--r-- | drivers/scsi/scsi_scan.c | 3 | ||||
-rw-r--r-- | include/linux/blk-mq.h | 8 | ||||
-rw-r--r-- | include/linux/blkdev.h | 12 | ||||
-rw-r--r-- | include/scsi/scsi_host.h | 3 | ||||
-rw-r--r-- | include/scsi/scsi_tcq.h | 3 |
16 files changed, 146 insertions, 62 deletions
diff --git a/block/blk-lib.c b/block/blk-lib.c index 8411be3c19d3..715e948f58a4 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c | |||
@@ -283,23 +283,45 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | |||
283 | * @sector: start sector | 283 | * @sector: start sector |
284 | * @nr_sects: number of sectors to write | 284 | * @nr_sects: number of sectors to write |
285 | * @gfp_mask: memory allocation flags (for bio_alloc) | 285 | * @gfp_mask: memory allocation flags (for bio_alloc) |
286 | * @discard: whether to discard the block range | ||
286 | * | 287 | * |
287 | * Description: | 288 | * Description: |
288 | * Generate and issue number of bios with zerofiled pages. | 289 | |
290 | * Zero-fill a block range. If the discard flag is set and the block | ||
291 | * device guarantees that subsequent READ operations to the block range | ||
292 | * in question will return zeroes, the blocks will be discarded. Should | ||
293 | * the discard request fail, if the discard flag is not set, or if | ||
294 | * discard_zeroes_data is not supported, this function will resort to | ||
295 | * zeroing the blocks manually, thus provisioning (allocating, | ||
296 | * anchoring) them. If the block device supports the WRITE SAME command | ||
297 | * blkdev_issue_zeroout() will use it to optimize the process of | ||
298 | * clearing the block range. Otherwise the zeroing will be performed | ||
299 | * using regular WRITE calls. | ||
289 | */ | 300 | */ |
290 | 301 | ||
291 | int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | 302 | int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, |
292 | sector_t nr_sects, gfp_t gfp_mask) | 303 | sector_t nr_sects, gfp_t gfp_mask, bool discard) |
293 | { | 304 | { |
305 | struct request_queue *q = bdev_get_queue(bdev); | ||
306 | unsigned char bdn[BDEVNAME_SIZE]; | ||
307 | |||
308 | if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data) { | ||
309 | |||
310 | if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0)) | ||
311 | return 0; | ||
312 | |||
313 | bdevname(bdev, bdn); | ||
314 | pr_warn("%s: DISCARD failed. Manually zeroing.\n", bdn); | ||
315 | } | ||
316 | |||
294 | if (bdev_write_same(bdev)) { | 317 | if (bdev_write_same(bdev)) { |
295 | unsigned char bdn[BDEVNAME_SIZE]; | ||
296 | 318 | ||
297 | if (!blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, | 319 | if (!blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, |
298 | ZERO_PAGE(0))) | 320 | ZERO_PAGE(0))) |
299 | return 0; | 321 | return 0; |
300 | 322 | ||
301 | bdevname(bdev, bdn); | 323 | bdevname(bdev, bdn); |
302 | pr_err("%s: WRITE SAME failed. Manually zeroing.\n", bdn); | 324 | pr_warn("%s: WRITE SAME failed. Manually zeroing.\n", bdn); |
303 | } | 325 | } |
304 | 326 | ||
305 | return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask); | 327 | return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask); |
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 60c9d4a93fe4..e3387a74a9a2 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c | |||
@@ -140,35 +140,39 @@ static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx, | |||
140 | return atomic_read(&hctx->nr_active) < depth; | 140 | return atomic_read(&hctx->nr_active) < depth; |
141 | } | 141 | } |
142 | 142 | ||
143 | static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag) | 143 | static int __bt_get_word(struct blk_align_bitmap *bm, unsigned int last_tag, |
144 | bool nowrap) | ||
144 | { | 145 | { |
145 | int tag, org_last_tag, end; | 146 | int tag, org_last_tag = last_tag; |
146 | bool wrap = last_tag != 0; | ||
147 | 147 | ||
148 | org_last_tag = last_tag; | 148 | while (1) { |
149 | end = bm->depth; | 149 | tag = find_next_zero_bit(&bm->word, bm->depth, last_tag); |
150 | do { | 150 | if (unlikely(tag >= bm->depth)) { |
151 | restart: | ||
152 | tag = find_next_zero_bit(&bm->word, end, last_tag); | ||
153 | if (unlikely(tag >= end)) { | ||
154 | /* | 151 | /* |
155 | * We started with an offset, start from 0 to | 152 | * We started with an offset, and we didn't reset the |
153 | * offset to 0 in a failure case, so start from 0 to | ||
156 | * exhaust the map. | 154 | * exhaust the map. |
157 | */ | 155 | */ |
158 | if (wrap) { | 156 | if (org_last_tag && last_tag && !nowrap) { |
159 | wrap = false; | 157 | last_tag = org_last_tag = 0; |
160 | end = org_last_tag; | 158 | continue; |
161 | last_tag = 0; | ||
162 | goto restart; | ||
163 | } | 159 | } |
164 | return -1; | 160 | return -1; |
165 | } | 161 | } |
162 | |||
163 | if (!test_and_set_bit(tag, &bm->word)) | ||
164 | break; | ||
165 | |||
166 | last_tag = tag + 1; | 166 | last_tag = tag + 1; |
167 | } while (test_and_set_bit(tag, &bm->word)); | 167 | if (last_tag >= bm->depth - 1) |
168 | last_tag = 0; | ||
169 | } | ||
168 | 170 | ||
169 | return tag; | 171 | return tag; |
170 | } | 172 | } |
171 | 173 | ||
174 | #define BT_ALLOC_RR(tags) (tags->alloc_policy == BLK_TAG_ALLOC_RR) | ||
175 | |||
172 | /* | 176 | /* |
173 | * Straight forward bitmap tag implementation, where each bit is a tag | 177 | * Straight forward bitmap tag implementation, where each bit is a tag |
174 | * (cleared == free, and set == busy). The small twist is using per-cpu | 178 | * (cleared == free, and set == busy). The small twist is using per-cpu |
@@ -181,7 +185,7 @@ restart: | |||
181 | * until the map is exhausted. | 185 | * until the map is exhausted. |
182 | */ | 186 | */ |
183 | static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt, | 187 | static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt, |
184 | unsigned int *tag_cache) | 188 | unsigned int *tag_cache, struct blk_mq_tags *tags) |
185 | { | 189 | { |
186 | unsigned int last_tag, org_last_tag; | 190 | unsigned int last_tag, org_last_tag; |
187 | int index, i, tag; | 191 | int index, i, tag; |
@@ -193,15 +197,24 @@ static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt, | |||
193 | index = TAG_TO_INDEX(bt, last_tag); | 197 | index = TAG_TO_INDEX(bt, last_tag); |
194 | 198 | ||
195 | for (i = 0; i < bt->map_nr; i++) { | 199 | for (i = 0; i < bt->map_nr; i++) { |
196 | tag = __bt_get_word(&bt->map[index], TAG_TO_BIT(bt, last_tag)); | 200 | tag = __bt_get_word(&bt->map[index], TAG_TO_BIT(bt, last_tag), |
201 | BT_ALLOC_RR(tags)); | ||
197 | if (tag != -1) { | 202 | if (tag != -1) { |
198 | tag += (index << bt->bits_per_word); | 203 | tag += (index << bt->bits_per_word); |
199 | goto done; | 204 | goto done; |
200 | } | 205 | } |
201 | 206 | ||
202 | last_tag = 0; | 207 | /* |
203 | if (++index >= bt->map_nr) | 208 | * Jump to next index, and reset the last tag to be the |
209 | * first tag of that index | ||
210 | */ | ||
211 | index++; | ||
212 | last_tag = (index << bt->bits_per_word); | ||
213 | |||
214 | if (index >= bt->map_nr) { | ||
204 | index = 0; | 215 | index = 0; |
216 | last_tag = 0; | ||
217 | } | ||
205 | } | 218 | } |
206 | 219 | ||
207 | *tag_cache = 0; | 220 | *tag_cache = 0; |
@@ -212,7 +225,7 @@ static int __bt_get(struct blk_mq_hw_ctx *hctx, struct blk_mq_bitmap_tags *bt, | |||
212 | * up using the specific cached tag. | 225 | * up using the specific cached tag. |
213 | */ | 226 | */ |
214 | done: | 227 | done: |
215 | if (tag == org_last_tag) { | 228 | if (tag == org_last_tag || unlikely(BT_ALLOC_RR(tags))) { |
216 | last_tag = tag + 1; | 229 | last_tag = tag + 1; |
217 | if (last_tag >= bt->depth - 1) | 230 | if (last_tag >= bt->depth - 1) |
218 | last_tag = 0; | 231 | last_tag = 0; |
@@ -241,13 +254,13 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt, | |||
241 | static int bt_get(struct blk_mq_alloc_data *data, | 254 | static int bt_get(struct blk_mq_alloc_data *data, |
242 | struct blk_mq_bitmap_tags *bt, | 255 | struct blk_mq_bitmap_tags *bt, |
243 | struct blk_mq_hw_ctx *hctx, | 256 | struct blk_mq_hw_ctx *hctx, |
244 | unsigned int *last_tag) | 257 | unsigned int *last_tag, struct blk_mq_tags *tags) |
245 | { | 258 | { |
246 | struct bt_wait_state *bs; | 259 | struct bt_wait_state *bs; |
247 | DEFINE_WAIT(wait); | 260 | DEFINE_WAIT(wait); |
248 | int tag; | 261 | int tag; |
249 | 262 | ||
250 | tag = __bt_get(hctx, bt, last_tag); | 263 | tag = __bt_get(hctx, bt, last_tag, tags); |
251 | if (tag != -1) | 264 | if (tag != -1) |
252 | return tag; | 265 | return tag; |
253 | 266 | ||
@@ -258,7 +271,7 @@ static int bt_get(struct blk_mq_alloc_data *data, | |||
258 | do { | 271 | do { |
259 | prepare_to_wait(&bs->wait, &wait, TASK_UNINTERRUPTIBLE); | 272 | prepare_to_wait(&bs->wait, &wait, TASK_UNINTERRUPTIBLE); |
260 | 273 | ||
261 | tag = __bt_get(hctx, bt, last_tag); | 274 | tag = __bt_get(hctx, bt, last_tag, tags); |
262 | if (tag != -1) | 275 | if (tag != -1) |
263 | break; | 276 | break; |
264 | 277 | ||
@@ -273,7 +286,7 @@ static int bt_get(struct blk_mq_alloc_data *data, | |||
273 | * Retry tag allocation after running the hardware queue, | 286 | * Retry tag allocation after running the hardware queue, |
274 | * as running the queue may also have found completions. | 287 | * as running the queue may also have found completions. |
275 | */ | 288 | */ |
276 | tag = __bt_get(hctx, bt, last_tag); | 289 | tag = __bt_get(hctx, bt, last_tag, tags); |
277 | if (tag != -1) | 290 | if (tag != -1) |
278 | break; | 291 | break; |
279 | 292 | ||
@@ -304,7 +317,7 @@ static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data) | |||
304 | int tag; | 317 | int tag; |
305 | 318 | ||
306 | tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx, | 319 | tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx, |
307 | &data->ctx->last_tag); | 320 | &data->ctx->last_tag, data->hctx->tags); |
308 | if (tag >= 0) | 321 | if (tag >= 0) |
309 | return tag + data->hctx->tags->nr_reserved_tags; | 322 | return tag + data->hctx->tags->nr_reserved_tags; |
310 | 323 | ||
@@ -320,7 +333,8 @@ static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data) | |||
320 | return BLK_MQ_TAG_FAIL; | 333 | return BLK_MQ_TAG_FAIL; |
321 | } | 334 | } |
322 | 335 | ||
323 | tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero); | 336 | tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero, |
337 | data->hctx->tags); | ||
324 | if (tag < 0) | 338 | if (tag < 0) |
325 | return BLK_MQ_TAG_FAIL; | 339 | return BLK_MQ_TAG_FAIL; |
326 | 340 | ||
@@ -392,7 +406,8 @@ void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, unsigned int tag, | |||
392 | 406 | ||
393 | BUG_ON(real_tag >= tags->nr_tags); | 407 | BUG_ON(real_tag >= tags->nr_tags); |
394 | bt_clear_tag(&tags->bitmap_tags, real_tag); | 408 | bt_clear_tag(&tags->bitmap_tags, real_tag); |
395 | *last_tag = real_tag; | 409 | if (likely(tags->alloc_policy == BLK_TAG_ALLOC_FIFO)) |
410 | *last_tag = real_tag; | ||
396 | } else { | 411 | } else { |
397 | BUG_ON(tag >= tags->nr_reserved_tags); | 412 | BUG_ON(tag >= tags->nr_reserved_tags); |
398 | bt_clear_tag(&tags->breserved_tags, tag); | 413 | bt_clear_tag(&tags->breserved_tags, tag); |
@@ -529,10 +544,12 @@ static void bt_free(struct blk_mq_bitmap_tags *bt) | |||
529 | } | 544 | } |
530 | 545 | ||
531 | static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags, | 546 | static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags, |
532 | int node) | 547 | int node, int alloc_policy) |
533 | { | 548 | { |
534 | unsigned int depth = tags->nr_tags - tags->nr_reserved_tags; | 549 | unsigned int depth = tags->nr_tags - tags->nr_reserved_tags; |
535 | 550 | ||
551 | tags->alloc_policy = alloc_policy; | ||
552 | |||
536 | if (bt_alloc(&tags->bitmap_tags, depth, node, false)) | 553 | if (bt_alloc(&tags->bitmap_tags, depth, node, false)) |
537 | goto enomem; | 554 | goto enomem; |
538 | if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, node, true)) | 555 | if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, node, true)) |
@@ -546,7 +563,8 @@ enomem: | |||
546 | } | 563 | } |
547 | 564 | ||
548 | struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, | 565 | struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, |
549 | unsigned int reserved_tags, int node) | 566 | unsigned int reserved_tags, |
567 | int node, int alloc_policy) | ||
550 | { | 568 | { |
551 | struct blk_mq_tags *tags; | 569 | struct blk_mq_tags *tags; |
552 | 570 | ||
@@ -562,7 +580,7 @@ struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, | |||
562 | tags->nr_tags = total_tags; | 580 | tags->nr_tags = total_tags; |
563 | tags->nr_reserved_tags = reserved_tags; | 581 | tags->nr_reserved_tags = reserved_tags; |
564 | 582 | ||
565 | return blk_mq_init_bitmap_tags(tags, node); | 583 | return blk_mq_init_bitmap_tags(tags, node, alloc_policy); |
566 | } | 584 | } |
567 | 585 | ||
568 | void blk_mq_free_tags(struct blk_mq_tags *tags) | 586 | void blk_mq_free_tags(struct blk_mq_tags *tags) |
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index a6fa0fc9d41a..90767b370308 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h | |||
@@ -42,10 +42,12 @@ struct blk_mq_tags { | |||
42 | 42 | ||
43 | struct request **rqs; | 43 | struct request **rqs; |
44 | struct list_head page_list; | 44 | struct list_head page_list; |
45 | |||
46 | int alloc_policy; | ||
45 | }; | 47 | }; |
46 | 48 | ||
47 | 49 | ||
48 | extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node); | 50 | extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags, unsigned int reserved_tags, int node, int alloc_policy); |
49 | extern void blk_mq_free_tags(struct blk_mq_tags *tags); | 51 | extern void blk_mq_free_tags(struct blk_mq_tags *tags); |
50 | 52 | ||
51 | extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data); | 53 | extern unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data); |
diff --git a/block/blk-mq.c b/block/blk-mq.c index a7d4a988516f..eb8e694fda06 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c | |||
@@ -1374,7 +1374,8 @@ static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set, | |||
1374 | size_t rq_size, left; | 1374 | size_t rq_size, left; |
1375 | 1375 | ||
1376 | tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags, | 1376 | tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags, |
1377 | set->numa_node); | 1377 | set->numa_node, |
1378 | BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); | ||
1378 | if (!tags) | 1379 | if (!tags) |
1379 | return NULL; | 1380 | return NULL; |
1380 | 1381 | ||
diff --git a/block/blk-tag.c b/block/blk-tag.c index a185b86741e5..f0344e6939d5 100644 --- a/block/blk-tag.c +++ b/block/blk-tag.c | |||
@@ -119,7 +119,7 @@ fail: | |||
119 | } | 119 | } |
120 | 120 | ||
121 | static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q, | 121 | static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q, |
122 | int depth) | 122 | int depth, int alloc_policy) |
123 | { | 123 | { |
124 | struct blk_queue_tag *tags; | 124 | struct blk_queue_tag *tags; |
125 | 125 | ||
@@ -131,6 +131,8 @@ static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q, | |||
131 | goto fail; | 131 | goto fail; |
132 | 132 | ||
133 | atomic_set(&tags->refcnt, 1); | 133 | atomic_set(&tags->refcnt, 1); |
134 | tags->alloc_policy = alloc_policy; | ||
135 | tags->next_tag = 0; | ||
134 | return tags; | 136 | return tags; |
135 | fail: | 137 | fail: |
136 | kfree(tags); | 138 | kfree(tags); |
@@ -140,10 +142,11 @@ fail: | |||
140 | /** | 142 | /** |
141 | * blk_init_tags - initialize the tag info for an external tag map | 143 | * blk_init_tags - initialize the tag info for an external tag map |
142 | * @depth: the maximum queue depth supported | 144 | * @depth: the maximum queue depth supported |
145 | * @alloc_policy: tag allocation policy | ||
143 | **/ | 146 | **/ |
144 | struct blk_queue_tag *blk_init_tags(int depth) | 147 | struct blk_queue_tag *blk_init_tags(int depth, int alloc_policy) |
145 | { | 148 | { |
146 | return __blk_queue_init_tags(NULL, depth); | 149 | return __blk_queue_init_tags(NULL, depth, alloc_policy); |
147 | } | 150 | } |
148 | EXPORT_SYMBOL(blk_init_tags); | 151 | EXPORT_SYMBOL(blk_init_tags); |
149 | 152 | ||
@@ -152,19 +155,20 @@ EXPORT_SYMBOL(blk_init_tags); | |||
152 | * @q: the request queue for the device | 155 | * @q: the request queue for the device |
153 | * @depth: the maximum queue depth supported | 156 | * @depth: the maximum queue depth supported |
154 | * @tags: the tag to use | 157 | * @tags: the tag to use |
158 | * @alloc_policy: tag allocation policy | ||
155 | * | 159 | * |
156 | * Queue lock must be held here if the function is called to resize an | 160 | * Queue lock must be held here if the function is called to resize an |
157 | * existing map. | 161 | * existing map. |
158 | **/ | 162 | **/ |
159 | int blk_queue_init_tags(struct request_queue *q, int depth, | 163 | int blk_queue_init_tags(struct request_queue *q, int depth, |
160 | struct blk_queue_tag *tags) | 164 | struct blk_queue_tag *tags, int alloc_policy) |
161 | { | 165 | { |
162 | int rc; | 166 | int rc; |
163 | 167 | ||
164 | BUG_ON(tags && q->queue_tags && tags != q->queue_tags); | 168 | BUG_ON(tags && q->queue_tags && tags != q->queue_tags); |
165 | 169 | ||
166 | if (!tags && !q->queue_tags) { | 170 | if (!tags && !q->queue_tags) { |
167 | tags = __blk_queue_init_tags(q, depth); | 171 | tags = __blk_queue_init_tags(q, depth, alloc_policy); |
168 | 172 | ||
169 | if (!tags) | 173 | if (!tags) |
170 | return -ENOMEM; | 174 | return -ENOMEM; |
@@ -344,9 +348,21 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq) | |||
344 | } | 348 | } |
345 | 349 | ||
346 | do { | 350 | do { |
347 | tag = find_first_zero_bit(bqt->tag_map, max_depth); | 351 | if (bqt->alloc_policy == BLK_TAG_ALLOC_FIFO) { |
348 | if (tag >= max_depth) | 352 | tag = find_first_zero_bit(bqt->tag_map, max_depth); |
349 | return 1; | 353 | if (tag >= max_depth) |
354 | return 1; | ||
355 | } else { | ||
356 | int start = bqt->next_tag; | ||
357 | int size = min_t(int, bqt->max_depth, max_depth + start); | ||
358 | tag = find_next_zero_bit(bqt->tag_map, size, start); | ||
359 | if (tag >= size && start + size > bqt->max_depth) { | ||
360 | size = start + size - bqt->max_depth; | ||
361 | tag = find_first_zero_bit(bqt->tag_map, size); | ||
362 | } | ||
363 | if (tag >= size) | ||
364 | return 1; | ||
365 | } | ||
350 | 366 | ||
351 | } while (test_and_set_bit_lock(tag, bqt->tag_map)); | 367 | } while (test_and_set_bit_lock(tag, bqt->tag_map)); |
352 | /* | 368 | /* |
@@ -354,6 +370,7 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq) | |||
354 | * See blk_queue_end_tag for details. | 370 | * See blk_queue_end_tag for details. |
355 | */ | 371 | */ |
356 | 372 | ||
373 | bqt->next_tag = (tag + 1) % bqt->max_depth; | ||
357 | rq->cmd_flags |= REQ_QUEUED; | 374 | rq->cmd_flags |= REQ_QUEUED; |
358 | rq->tag = tag; | 375 | rq->tag = tag; |
359 | bqt->tag_index[tag] = rq; | 376 | bqt->tag_index[tag] = rq; |
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 6f2751d305de..b9abdca84c17 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c | |||
@@ -3656,12 +3656,17 @@ static struct cfq_queue * | |||
3656 | cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, | 3656 | cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, |
3657 | struct bio *bio, gfp_t gfp_mask) | 3657 | struct bio *bio, gfp_t gfp_mask) |
3658 | { | 3658 | { |
3659 | const int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio); | 3659 | int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio); |
3660 | const int ioprio = IOPRIO_PRIO_DATA(cic->ioprio); | 3660 | int ioprio = IOPRIO_PRIO_DATA(cic->ioprio); |
3661 | struct cfq_queue **async_cfqq = NULL; | 3661 | struct cfq_queue **async_cfqq = NULL; |
3662 | struct cfq_queue *cfqq = NULL; | 3662 | struct cfq_queue *cfqq = NULL; |
3663 | 3663 | ||
3664 | if (!is_sync) { | 3664 | if (!is_sync) { |
3665 | if (!ioprio_valid(cic->ioprio)) { | ||
3666 | struct task_struct *tsk = current; | ||
3667 | ioprio = task_nice_ioprio(tsk); | ||
3668 | ioprio_class = task_nice_ioclass(tsk); | ||
3669 | } | ||
3665 | async_cfqq = cfq_async_queue_prio(cfqd, ioprio_class, ioprio); | 3670 | async_cfqq = cfq_async_queue_prio(cfqd, ioprio_class, ioprio); |
3666 | cfqq = *async_cfqq; | 3671 | cfqq = *async_cfqq; |
3667 | } | 3672 | } |
diff --git a/block/ioctl.c b/block/ioctl.c index 6c7bf903742f..7d8befde2aca 100644 --- a/block/ioctl.c +++ b/block/ioctl.c | |||
@@ -198,7 +198,7 @@ static int blk_ioctl_zeroout(struct block_device *bdev, uint64_t start, | |||
198 | if (start + len > (i_size_read(bdev->bd_inode) >> 9)) | 198 | if (start + len > (i_size_read(bdev->bd_inode) >> 9)) |
199 | return -EINVAL; | 199 | return -EINVAL; |
200 | 200 | ||
201 | return blkdev_issue_zeroout(bdev, start, len, GFP_KERNEL); | 201 | return blkdev_issue_zeroout(bdev, start, len, GFP_KERNEL, false); |
202 | } | 202 | } |
203 | 203 | ||
204 | static int put_ushort(unsigned long arg, unsigned short val) | 204 | static int put_ushort(unsigned long arg, unsigned short val) |
diff --git a/block/partitions/check.c b/block/partitions/check.c index 9ac1df74f699..16118d11dbfc 100644 --- a/block/partitions/check.c +++ b/block/partitions/check.c | |||
@@ -184,12 +184,12 @@ check_partition(struct gendisk *hd, struct block_device *bdev) | |||
184 | if (err) | 184 | if (err) |
185 | /* The partition is unrecognized. So report I/O errors if there were any */ | 185 | /* The partition is unrecognized. So report I/O errors if there were any */ |
186 | res = err; | 186 | res = err; |
187 | if (!res) | 187 | if (res) { |
188 | strlcat(state->pp_buf, " unknown partition table\n", PAGE_SIZE); | 188 | if (warn_no_part) |
189 | else if (warn_no_part) | 189 | strlcat(state->pp_buf, |
190 | strlcat(state->pp_buf, " unable to read partition table\n", PAGE_SIZE); | 190 | " unable to read partition table\n", PAGE_SIZE); |
191 | 191 | printk(KERN_INFO "%s", state->pp_buf); | |
192 | printk(KERN_INFO "%s", state->pp_buf); | 192 | } |
193 | 193 | ||
194 | free_page((unsigned long)state->pp_buf); | 194 | free_page((unsigned long)state->pp_buf); |
195 | free_partitions(state); | 195 | free_partitions(state); |
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index d169b4a79267..cee20354ac37 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c | |||
@@ -1388,7 +1388,7 @@ int drbd_submit_peer_request(struct drbd_device *device, | |||
1388 | list_add_tail(&peer_req->w.list, &device->active_ee); | 1388 | list_add_tail(&peer_req->w.list, &device->active_ee); |
1389 | spin_unlock_irq(&device->resource->req_lock); | 1389 | spin_unlock_irq(&device->resource->req_lock); |
1390 | if (blkdev_issue_zeroout(device->ldev->backing_bdev, | 1390 | if (blkdev_issue_zeroout(device->ldev->backing_bdev, |
1391 | sector, data_size >> 9, GFP_NOIO)) | 1391 | sector, data_size >> 9, GFP_NOIO, false)) |
1392 | peer_req->flags |= EE_WAS_ERROR; | 1392 | peer_req->flags |= EE_WAS_ERROR; |
1393 | drbd_endio_write_sec_final(peer_req); | 1393 | drbd_endio_write_sec_final(peer_req); |
1394 | return 0; | 1394 | return 0; |
diff --git a/drivers/block/osdblk.c b/drivers/block/osdblk.c index 79aa179305b5..e22942596207 100644 --- a/drivers/block/osdblk.c +++ b/drivers/block/osdblk.c | |||
@@ -423,7 +423,7 @@ static int osdblk_init_disk(struct osdblk_device *osdev) | |||
423 | } | 423 | } |
424 | 424 | ||
425 | /* switch queue to TCQ mode; allocate tag map */ | 425 | /* switch queue to TCQ mode; allocate tag map */ |
426 | rc = blk_queue_init_tags(q, OSDBLK_MAX_REQ, NULL); | 426 | rc = blk_queue_init_tags(q, OSDBLK_MAX_REQ, NULL, BLK_TAG_ALLOC_FIFO); |
427 | if (rc) { | 427 | if (rc) { |
428 | blk_cleanup_queue(q); | 428 | blk_cleanup_queue(q); |
429 | put_disk(disk); | 429 | put_disk(disk); |
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 9ea95dd3e260..49ab11508286 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c | |||
@@ -2188,6 +2188,8 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) | |||
2188 | shost->tag_set.cmd_size = cmd_size; | 2188 | shost->tag_set.cmd_size = cmd_size; |
2189 | shost->tag_set.numa_node = NUMA_NO_NODE; | 2189 | shost->tag_set.numa_node = NUMA_NO_NODE; |
2190 | shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; | 2190 | shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; |
2191 | shost->tag_set.flags |= | ||
2192 | BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy); | ||
2191 | shost->tag_set.driver_data = shost; | 2193 | shost->tag_set.driver_data = shost; |
2192 | 2194 | ||
2193 | return blk_mq_alloc_tag_set(&shost->tag_set); | 2195 | return blk_mq_alloc_tag_set(&shost->tag_set); |
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index 983aed10ff2f..921a8c897eb2 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c | |||
@@ -290,7 +290,8 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, | |||
290 | if (!shost_use_blk_mq(sdev->host) && | 290 | if (!shost_use_blk_mq(sdev->host) && |
291 | (shost->bqt || shost->hostt->use_blk_tags)) { | 291 | (shost->bqt || shost->hostt->use_blk_tags)) { |
292 | blk_queue_init_tags(sdev->request_queue, | 292 | blk_queue_init_tags(sdev->request_queue, |
293 | sdev->host->cmd_per_lun, shost->bqt); | 293 | sdev->host->cmd_per_lun, shost->bqt, |
294 | shost->hostt->tag_alloc_policy); | ||
294 | } | 295 | } |
295 | scsi_change_queue_depth(sdev, sdev->host->cmd_per_lun); | 296 | scsi_change_queue_depth(sdev, sdev->host->cmd_per_lun); |
296 | 297 | ||
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 5b6500c77ed2..86b08b1a5eba 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h | |||
@@ -147,6 +147,8 @@ enum { | |||
147 | BLK_MQ_F_SG_MERGE = 1 << 2, | 147 | BLK_MQ_F_SG_MERGE = 1 << 2, |
148 | BLK_MQ_F_SYSFS_UP = 1 << 3, | 148 | BLK_MQ_F_SYSFS_UP = 1 << 3, |
149 | BLK_MQ_F_DEFER_ISSUE = 1 << 4, | 149 | BLK_MQ_F_DEFER_ISSUE = 1 << 4, |
150 | BLK_MQ_F_ALLOC_POLICY_START_BIT = 8, | ||
151 | BLK_MQ_F_ALLOC_POLICY_BITS = 1, | ||
150 | 152 | ||
151 | BLK_MQ_S_STOPPED = 0, | 153 | BLK_MQ_S_STOPPED = 0, |
152 | BLK_MQ_S_TAG_ACTIVE = 1, | 154 | BLK_MQ_S_TAG_ACTIVE = 1, |
@@ -155,6 +157,12 @@ enum { | |||
155 | 157 | ||
156 | BLK_MQ_CPU_WORK_BATCH = 8, | 158 | BLK_MQ_CPU_WORK_BATCH = 8, |
157 | }; | 159 | }; |
160 | #define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \ | ||
161 | ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \ | ||
162 | ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) | ||
163 | #define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \ | ||
164 | ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \ | ||
165 | << BLK_MQ_F_ALLOC_POLICY_START_BIT) | ||
158 | 166 | ||
159 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); | 167 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); |
160 | void blk_mq_finish_init(struct request_queue *q); | 168 | void blk_mq_finish_init(struct request_queue *q); |
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index e9086be6d9a0..6f388fd1c11c 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h | |||
@@ -272,7 +272,11 @@ struct blk_queue_tag { | |||
272 | int max_depth; /* what we will send to device */ | 272 | int max_depth; /* what we will send to device */ |
273 | int real_max_depth; /* what the array can hold */ | 273 | int real_max_depth; /* what the array can hold */ |
274 | atomic_t refcnt; /* map can be shared */ | 274 | atomic_t refcnt; /* map can be shared */ |
275 | int alloc_policy; /* tag allocation policy */ | ||
276 | int next_tag; /* next tag */ | ||
275 | }; | 277 | }; |
278 | #define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */ | ||
279 | #define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */ | ||
276 | 280 | ||
277 | #define BLK_SCSI_MAX_CMDS (256) | 281 | #define BLK_SCSI_MAX_CMDS (256) |
278 | #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8)) | 282 | #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8)) |
@@ -1139,11 +1143,11 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) | |||
1139 | extern int blk_queue_start_tag(struct request_queue *, struct request *); | 1143 | extern int blk_queue_start_tag(struct request_queue *, struct request *); |
1140 | extern struct request *blk_queue_find_tag(struct request_queue *, int); | 1144 | extern struct request *blk_queue_find_tag(struct request_queue *, int); |
1141 | extern void blk_queue_end_tag(struct request_queue *, struct request *); | 1145 | extern void blk_queue_end_tag(struct request_queue *, struct request *); |
1142 | extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *); | 1146 | extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int); |
1143 | extern void blk_queue_free_tags(struct request_queue *); | 1147 | extern void blk_queue_free_tags(struct request_queue *); |
1144 | extern int blk_queue_resize_tags(struct request_queue *, int); | 1148 | extern int blk_queue_resize_tags(struct request_queue *, int); |
1145 | extern void blk_queue_invalidate_tags(struct request_queue *); | 1149 | extern void blk_queue_invalidate_tags(struct request_queue *); |
1146 | extern struct blk_queue_tag *blk_init_tags(int); | 1150 | extern struct blk_queue_tag *blk_init_tags(int, int); |
1147 | extern void blk_free_tags(struct blk_queue_tag *); | 1151 | extern void blk_free_tags(struct blk_queue_tag *); |
1148 | 1152 | ||
1149 | static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, | 1153 | static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, |
@@ -1162,7 +1166,7 @@ extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | |||
1162 | extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, | 1166 | extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, |
1163 | sector_t nr_sects, gfp_t gfp_mask, struct page *page); | 1167 | sector_t nr_sects, gfp_t gfp_mask, struct page *page); |
1164 | extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | 1168 | extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, |
1165 | sector_t nr_sects, gfp_t gfp_mask); | 1169 | sector_t nr_sects, gfp_t gfp_mask, bool discard); |
1166 | static inline int sb_issue_discard(struct super_block *sb, sector_t block, | 1170 | static inline int sb_issue_discard(struct super_block *sb, sector_t block, |
1167 | sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) | 1171 | sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) |
1168 | { | 1172 | { |
@@ -1176,7 +1180,7 @@ static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, | |||
1176 | return blkdev_issue_zeroout(sb->s_bdev, | 1180 | return blkdev_issue_zeroout(sb->s_bdev, |
1177 | block << (sb->s_blocksize_bits - 9), | 1181 | block << (sb->s_blocksize_bits - 9), |
1178 | nr_blocks << (sb->s_blocksize_bits - 9), | 1182 | nr_blocks << (sb->s_blocksize_bits - 9), |
1179 | gfp_mask); | 1183 | gfp_mask, true); |
1180 | } | 1184 | } |
1181 | 1185 | ||
1182 | extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); | 1186 | extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); |
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 019e66858ce6..e113c757d555 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h | |||
@@ -402,6 +402,9 @@ struct scsi_host_template { | |||
402 | */ | 402 | */ |
403 | unsigned char present; | 403 | unsigned char present; |
404 | 404 | ||
405 | /* If use block layer to manage tags, this is tag allocation policy */ | ||
406 | int tag_alloc_policy; | ||
407 | |||
405 | /* | 408 | /* |
406 | * Let the block layer assigns tags to all commands. | 409 | * Let the block layer assigns tags to all commands. |
407 | */ | 410 | */ |
diff --git a/include/scsi/scsi_tcq.h b/include/scsi/scsi_tcq.h index 9708b28bd2aa..b27977e8aaed 100644 --- a/include/scsi/scsi_tcq.h +++ b/include/scsi/scsi_tcq.h | |||
@@ -66,7 +66,8 @@ static inline int scsi_init_shared_tag_map(struct Scsi_Host *shost, int depth) | |||
66 | * devices on the shared host (for libata) | 66 | * devices on the shared host (for libata) |
67 | */ | 67 | */ |
68 | if (!shost->bqt) { | 68 | if (!shost->bqt) { |
69 | shost->bqt = blk_init_tags(depth); | 69 | shost->bqt = blk_init_tags(depth, |
70 | shost->hostt->tag_alloc_policy); | ||
70 | if (!shost->bqt) | 71 | if (!shost->bqt) |
71 | return -ENOMEM; | 72 | return -ENOMEM; |
72 | } | 73 | } |