aboutsummaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
Diffstat (limited to 'block')
-rw-r--r--block/Kconfig11
-rw-r--r--block/as-iosched.c24
-rw-r--r--block/blk-barrier.c19
-rw-r--r--block/blk-core.c740
-rw-r--r--block/blk-exec.c1
-rw-r--r--block/blk-map.c21
-rw-r--r--block/blk-merge.c46
-rw-r--r--block/blk-tag.c17
-rw-r--r--block/blk-timeout.c22
-rw-r--r--block/blk.h51
-rw-r--r--block/bsg.c8
-rw-r--r--block/cfq-iosched.c36
-rw-r--r--block/deadline-iosched.c2
-rw-r--r--block/elevator.c167
-rw-r--r--block/scsi_ioctl.c5
15 files changed, 599 insertions, 571 deletions
diff --git a/block/Kconfig b/block/Kconfig
index e7d12782bcfb..2c39527aa7db 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -26,6 +26,7 @@ if BLOCK
26config LBD 26config LBD
27 bool "Support for large block devices and files" 27 bool "Support for large block devices and files"
28 depends on !64BIT 28 depends on !64BIT
29 default y
29 help 30 help
30 Enable block devices or files of size 2TB and larger. 31 Enable block devices or files of size 2TB and larger.
31 32
@@ -38,11 +39,13 @@ config LBD
38 39
39 The ext4 filesystem requires that this feature be enabled in 40 The ext4 filesystem requires that this feature be enabled in
40 order to support filesystems that have the huge_file feature 41 order to support filesystems that have the huge_file feature
41 enabled. Otherwise, it will refuse to mount any filesystems 42 enabled. Otherwise, it will refuse to mount in the read-write
42 that use the huge_file feature, which is enabled by default 43 mode any filesystems that use the huge_file feature, which is
43 by mke2fs.ext4. The GFS2 filesystem also requires this feature. 44 enabled by default by mke2fs.ext4.
44 45
45 If unsure, say N. 46 The GFS2 filesystem also requires this feature.
47
48 If unsure, say Y.
46 49
47config BLK_DEV_BSG 50config BLK_DEV_BSG
48 bool "Block layer SG support v4 (EXPERIMENTAL)" 51 bool "Block layer SG support v4 (EXPERIMENTAL)"
diff --git a/block/as-iosched.c b/block/as-iosched.c
index c48fa670d221..7a12cf6ee1d3 100644
--- a/block/as-iosched.c
+++ b/block/as-iosched.c
@@ -306,8 +306,8 @@ as_choose_req(struct as_data *ad, struct request *rq1, struct request *rq2)
306 data_dir = rq_is_sync(rq1); 306 data_dir = rq_is_sync(rq1);
307 307
308 last = ad->last_sector[data_dir]; 308 last = ad->last_sector[data_dir];
309 s1 = rq1->sector; 309 s1 = blk_rq_pos(rq1);
310 s2 = rq2->sector; 310 s2 = blk_rq_pos(rq2);
311 311
312 BUG_ON(data_dir != rq_is_sync(rq2)); 312 BUG_ON(data_dir != rq_is_sync(rq2));
313 313
@@ -566,13 +566,15 @@ static void as_update_iohist(struct as_data *ad, struct as_io_context *aic,
566 as_update_thinktime(ad, aic, thinktime); 566 as_update_thinktime(ad, aic, thinktime);
567 567
568 /* Calculate read -> read seek distance */ 568 /* Calculate read -> read seek distance */
569 if (aic->last_request_pos < rq->sector) 569 if (aic->last_request_pos < blk_rq_pos(rq))
570 seek_dist = rq->sector - aic->last_request_pos; 570 seek_dist = blk_rq_pos(rq) -
571 aic->last_request_pos;
571 else 572 else
572 seek_dist = aic->last_request_pos - rq->sector; 573 seek_dist = aic->last_request_pos -
574 blk_rq_pos(rq);
573 as_update_seekdist(ad, aic, seek_dist); 575 as_update_seekdist(ad, aic, seek_dist);
574 } 576 }
575 aic->last_request_pos = rq->sector + rq->nr_sectors; 577 aic->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
576 set_bit(AS_TASK_IOSTARTED, &aic->state); 578 set_bit(AS_TASK_IOSTARTED, &aic->state);
577 spin_unlock(&aic->lock); 579 spin_unlock(&aic->lock);
578 } 580 }
@@ -587,7 +589,7 @@ static int as_close_req(struct as_data *ad, struct as_io_context *aic,
587{ 589{
588 unsigned long delay; /* jiffies */ 590 unsigned long delay; /* jiffies */
589 sector_t last = ad->last_sector[ad->batch_data_dir]; 591 sector_t last = ad->last_sector[ad->batch_data_dir];
590 sector_t next = rq->sector; 592 sector_t next = blk_rq_pos(rq);
591 sector_t delta; /* acceptable close offset (in sectors) */ 593 sector_t delta; /* acceptable close offset (in sectors) */
592 sector_t s; 594 sector_t s;
593 595
@@ -981,7 +983,7 @@ static void as_move_to_dispatch(struct as_data *ad, struct request *rq)
981 * This has to be set in order to be correctly updated by 983 * This has to be set in order to be correctly updated by
982 * as_find_next_rq 984 * as_find_next_rq
983 */ 985 */
984 ad->last_sector[data_dir] = rq->sector + rq->nr_sectors; 986 ad->last_sector[data_dir] = blk_rq_pos(rq) + blk_rq_sectors(rq);
985 987
986 if (data_dir == BLK_RW_SYNC) { 988 if (data_dir == BLK_RW_SYNC) {
987 struct io_context *ioc = RQ_IOC(rq); 989 struct io_context *ioc = RQ_IOC(rq);
@@ -1312,12 +1314,8 @@ static void as_merged_requests(struct request_queue *q, struct request *req,
1312static void as_work_handler(struct work_struct *work) 1314static void as_work_handler(struct work_struct *work)
1313{ 1315{
1314 struct as_data *ad = container_of(work, struct as_data, antic_work); 1316 struct as_data *ad = container_of(work, struct as_data, antic_work);
1315 struct request_queue *q = ad->q;
1316 unsigned long flags;
1317 1317
1318 spin_lock_irqsave(q->queue_lock, flags); 1318 blk_run_queue(ad->q);
1319 blk_start_queueing(q);
1320 spin_unlock_irqrestore(q->queue_lock, flags);
1321} 1319}
1322 1320
1323static int as_may_queue(struct request_queue *q, int rw) 1321static int as_may_queue(struct request_queue *q, int rw)
diff --git a/block/blk-barrier.c b/block/blk-barrier.c
index 20b4111fa050..0d98054cdbd7 100644
--- a/block/blk-barrier.c
+++ b/block/blk-barrier.c
@@ -106,10 +106,7 @@ bool blk_ordered_complete_seq(struct request_queue *q, unsigned seq, int error)
106 */ 106 */
107 q->ordseq = 0; 107 q->ordseq = 0;
108 rq = q->orig_bar_rq; 108 rq = q->orig_bar_rq;
109 109 __blk_end_request_all(rq, q->orderr);
110 if (__blk_end_request(rq, q->orderr, blk_rq_bytes(rq)))
111 BUG();
112
113 return true; 110 return true;
114} 111}
115 112
@@ -166,7 +163,7 @@ static inline bool start_ordered(struct request_queue *q, struct request **rqp)
166 * For an empty barrier, there's no actual BAR request, which 163 * For an empty barrier, there's no actual BAR request, which
167 * in turn makes POSTFLUSH unnecessary. Mask them off. 164 * in turn makes POSTFLUSH unnecessary. Mask them off.
168 */ 165 */
169 if (!rq->hard_nr_sectors) { 166 if (!blk_rq_sectors(rq)) {
170 q->ordered &= ~(QUEUE_ORDERED_DO_BAR | 167 q->ordered &= ~(QUEUE_ORDERED_DO_BAR |
171 QUEUE_ORDERED_DO_POSTFLUSH); 168 QUEUE_ORDERED_DO_POSTFLUSH);
172 /* 169 /*
@@ -183,7 +180,7 @@ static inline bool start_ordered(struct request_queue *q, struct request **rqp)
183 } 180 }
184 181
185 /* stash away the original request */ 182 /* stash away the original request */
186 elv_dequeue_request(q, rq); 183 blk_dequeue_request(rq);
187 q->orig_bar_rq = rq; 184 q->orig_bar_rq = rq;
188 rq = NULL; 185 rq = NULL;
189 186
@@ -221,7 +218,7 @@ static inline bool start_ordered(struct request_queue *q, struct request **rqp)
221 } else 218 } else
222 skip |= QUEUE_ORDSEQ_PREFLUSH; 219 skip |= QUEUE_ORDSEQ_PREFLUSH;
223 220
224 if ((q->ordered & QUEUE_ORDERED_BY_DRAIN) && q->in_flight) 221 if ((q->ordered & QUEUE_ORDERED_BY_DRAIN) && queue_in_flight(q))
225 rq = NULL; 222 rq = NULL;
226 else 223 else
227 skip |= QUEUE_ORDSEQ_DRAIN; 224 skip |= QUEUE_ORDSEQ_DRAIN;
@@ -251,10 +248,8 @@ bool blk_do_ordered(struct request_queue *q, struct request **rqp)
251 * Queue ordering not supported. Terminate 248 * Queue ordering not supported. Terminate
252 * with prejudice. 249 * with prejudice.
253 */ 250 */
254 elv_dequeue_request(q, rq); 251 blk_dequeue_request(rq);
255 if (__blk_end_request(rq, -EOPNOTSUPP, 252 __blk_end_request_all(rq, -EOPNOTSUPP);
256 blk_rq_bytes(rq)))
257 BUG();
258 *rqp = NULL; 253 *rqp = NULL;
259 return false; 254 return false;
260 } 255 }
@@ -329,7 +324,7 @@ int blkdev_issue_flush(struct block_device *bdev, sector_t *error_sector)
329 /* 324 /*
330 * The driver must store the error location in ->bi_sector, if 325 * The driver must store the error location in ->bi_sector, if
331 * it supports it. For non-stacked drivers, this should be copied 326 * it supports it. For non-stacked drivers, this should be copied
332 * from rq->sector. 327 * from blk_rq_pos(rq).
333 */ 328 */
334 if (error_sector) 329 if (error_sector)
335 *error_sector = bio->bi_sector; 330 *error_sector = bio->bi_sector;
diff --git a/block/blk-core.c b/block/blk-core.c
index c89883be8737..59c4af523112 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -68,11 +68,11 @@ static void drive_stat_acct(struct request *rq, int new_io)
68 int rw = rq_data_dir(rq); 68 int rw = rq_data_dir(rq);
69 int cpu; 69 int cpu;
70 70
71 if (!blk_fs_request(rq) || !blk_do_io_stat(rq)) 71 if (!blk_do_io_stat(rq))
72 return; 72 return;
73 73
74 cpu = part_stat_lock(); 74 cpu = part_stat_lock();
75 part = disk_map_sector_rcu(rq->rq_disk, rq->sector); 75 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
76 76
77 if (!new_io) 77 if (!new_io)
78 part_stat_inc(cpu, part, merges[rw]); 78 part_stat_inc(cpu, part, merges[rw]);
@@ -127,13 +127,14 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
127 INIT_LIST_HEAD(&rq->timeout_list); 127 INIT_LIST_HEAD(&rq->timeout_list);
128 rq->cpu = -1; 128 rq->cpu = -1;
129 rq->q = q; 129 rq->q = q;
130 rq->sector = rq->hard_sector = (sector_t) -1; 130 rq->__sector = (sector_t) -1;
131 INIT_HLIST_NODE(&rq->hash); 131 INIT_HLIST_NODE(&rq->hash);
132 RB_CLEAR_NODE(&rq->rb_node); 132 RB_CLEAR_NODE(&rq->rb_node);
133 rq->cmd = rq->__cmd; 133 rq->cmd = rq->__cmd;
134 rq->cmd_len = BLK_MAX_CDB; 134 rq->cmd_len = BLK_MAX_CDB;
135 rq->tag = -1; 135 rq->tag = -1;
136 rq->ref_count = 1; 136 rq->ref_count = 1;
137 rq->start_time = jiffies;
137} 138}
138EXPORT_SYMBOL(blk_rq_init); 139EXPORT_SYMBOL(blk_rq_init);
139 140
@@ -184,14 +185,11 @@ void blk_dump_rq_flags(struct request *rq, char *msg)
184 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type, 185 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
185 rq->cmd_flags); 186 rq->cmd_flags);
186 187
187 printk(KERN_INFO " sector %llu, nr/cnr %lu/%u\n", 188 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
188 (unsigned long long)rq->sector, 189 (unsigned long long)blk_rq_pos(rq),
189 rq->nr_sectors, 190 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
190 rq->current_nr_sectors); 191 printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n",
191 printk(KERN_INFO " bio %p, biotail %p, buffer %p, data %p, len %u\n", 192 rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
192 rq->bio, rq->biotail,
193 rq->buffer, rq->data,
194 rq->data_len);
195 193
196 if (blk_pc_request(rq)) { 194 if (blk_pc_request(rq)) {
197 printk(KERN_INFO " cdb: "); 195 printk(KERN_INFO " cdb: ");
@@ -333,24 +331,6 @@ void blk_unplug(struct request_queue *q)
333} 331}
334EXPORT_SYMBOL(blk_unplug); 332EXPORT_SYMBOL(blk_unplug);
335 333
336static void blk_invoke_request_fn(struct request_queue *q)
337{
338 if (unlikely(blk_queue_stopped(q)))
339 return;
340
341 /*
342 * one level of recursion is ok and is much faster than kicking
343 * the unplug handling
344 */
345 if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
346 q->request_fn(q);
347 queue_flag_clear(QUEUE_FLAG_REENTER, q);
348 } else {
349 queue_flag_set(QUEUE_FLAG_PLUGGED, q);
350 kblockd_schedule_work(q, &q->unplug_work);
351 }
352}
353
354/** 334/**
355 * blk_start_queue - restart a previously stopped queue 335 * blk_start_queue - restart a previously stopped queue
356 * @q: The &struct request_queue in question 336 * @q: The &struct request_queue in question
@@ -365,7 +345,7 @@ void blk_start_queue(struct request_queue *q)
365 WARN_ON(!irqs_disabled()); 345 WARN_ON(!irqs_disabled());
366 346
367 queue_flag_clear(QUEUE_FLAG_STOPPED, q); 347 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
368 blk_invoke_request_fn(q); 348 __blk_run_queue(q);
369} 349}
370EXPORT_SYMBOL(blk_start_queue); 350EXPORT_SYMBOL(blk_start_queue);
371 351
@@ -425,12 +405,23 @@ void __blk_run_queue(struct request_queue *q)
425{ 405{
426 blk_remove_plug(q); 406 blk_remove_plug(q);
427 407
408 if (unlikely(blk_queue_stopped(q)))
409 return;
410
411 if (elv_queue_empty(q))
412 return;
413
428 /* 414 /*
429 * Only recurse once to avoid overrunning the stack, let the unplug 415 * Only recurse once to avoid overrunning the stack, let the unplug
430 * handling reinvoke the handler shortly if we already got there. 416 * handling reinvoke the handler shortly if we already got there.
431 */ 417 */
432 if (!elv_queue_empty(q)) 418 if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
433 blk_invoke_request_fn(q); 419 q->request_fn(q);
420 queue_flag_clear(QUEUE_FLAG_REENTER, q);
421 } else {
422 queue_flag_set(QUEUE_FLAG_PLUGGED, q);
423 kblockd_schedule_work(q, &q->unplug_work);
424 }
434} 425}
435EXPORT_SYMBOL(__blk_run_queue); 426EXPORT_SYMBOL(__blk_run_queue);
436 427
@@ -440,9 +431,7 @@ EXPORT_SYMBOL(__blk_run_queue);
440 * 431 *
441 * Description: 432 * Description:
442 * Invoke request handling on this queue, if it has pending work to do. 433 * Invoke request handling on this queue, if it has pending work to do.
443 * May be used to restart queueing when a request has completed. Also 434 * May be used to restart queueing when a request has completed.
444 * See @blk_start_queueing.
445 *
446 */ 435 */
447void blk_run_queue(struct request_queue *q) 436void blk_run_queue(struct request_queue *q)
448{ 437{
@@ -902,26 +891,58 @@ struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
902EXPORT_SYMBOL(blk_get_request); 891EXPORT_SYMBOL(blk_get_request);
903 892
904/** 893/**
905 * blk_start_queueing - initiate dispatch of requests to device 894 * blk_make_request - given a bio, allocate a corresponding struct request.
906 * @q: request queue to kick into gear 895 *
896 * @bio: The bio describing the memory mappings that will be submitted for IO.
897 * It may be a chained-bio properly constructed by block/bio layer.
898 *
899 * blk_make_request is the parallel of generic_make_request for BLOCK_PC
900 * type commands. Where the struct request needs to be farther initialized by
901 * the caller. It is passed a &struct bio, which describes the memory info of
902 * the I/O transfer.
907 * 903 *
908 * This is basically a helper to remove the need to know whether a queue 904 * The caller of blk_make_request must make sure that bi_io_vec
909 * is plugged or not if someone just wants to initiate dispatch of requests 905 * are set to describe the memory buffers. That bio_data_dir() will return
910 * for this queue. Should be used to start queueing on a device outside 906 * the needed direction of the request. (And all bio's in the passed bio-chain
911 * of ->request_fn() context. Also see @blk_run_queue. 907 * are properly set accordingly)
912 * 908 *
913 * The queue lock must be held with interrupts disabled. 909 * If called under none-sleepable conditions, mapped bio buffers must not
910 * need bouncing, by calling the appropriate masked or flagged allocator,
911 * suitable for the target device. Otherwise the call to blk_queue_bounce will
912 * BUG.
913 *
914 * WARNING: When allocating/cloning a bio-chain, careful consideration should be
915 * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
916 * anything but the first bio in the chain. Otherwise you risk waiting for IO
917 * completion of a bio that hasn't been submitted yet, thus resulting in a
918 * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
919 * of bio_alloc(), as that avoids the mempool deadlock.
920 * If possible a big IO should be split into smaller parts when allocation
921 * fails. Partial allocation should not be an error, or you risk a live-lock.
914 */ 922 */
915void blk_start_queueing(struct request_queue *q) 923struct request *blk_make_request(struct request_queue *q, struct bio *bio,
924 gfp_t gfp_mask)
916{ 925{
917 if (!blk_queue_plugged(q)) { 926 struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
918 if (unlikely(blk_queue_stopped(q))) 927
919 return; 928 if (unlikely(!rq))
920 q->request_fn(q); 929 return ERR_PTR(-ENOMEM);
921 } else 930
922 __generic_unplug_device(q); 931 for_each_bio(bio) {
932 struct bio *bounce_bio = bio;
933 int ret;
934
935 blk_queue_bounce(q, &bounce_bio);
936 ret = blk_rq_append_bio(q, rq, bounce_bio);
937 if (unlikely(ret)) {
938 blk_put_request(rq);
939 return ERR_PTR(ret);
940 }
941 }
942
943 return rq;
923} 944}
924EXPORT_SYMBOL(blk_start_queueing); 945EXPORT_SYMBOL(blk_make_request);
925 946
926/** 947/**
927 * blk_requeue_request - put a request back on queue 948 * blk_requeue_request - put a request back on queue
@@ -935,6 +956,8 @@ EXPORT_SYMBOL(blk_start_queueing);
935 */ 956 */
936void blk_requeue_request(struct request_queue *q, struct request *rq) 957void blk_requeue_request(struct request_queue *q, struct request *rq)
937{ 958{
959 BUG_ON(blk_queued_rq(rq));
960
938 blk_delete_timer(rq); 961 blk_delete_timer(rq);
939 blk_clear_rq_complete(rq); 962 blk_clear_rq_complete(rq);
940 trace_block_rq_requeue(q, rq); 963 trace_block_rq_requeue(q, rq);
@@ -977,7 +1000,6 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
977 * barrier 1000 * barrier
978 */ 1001 */
979 rq->cmd_type = REQ_TYPE_SPECIAL; 1002 rq->cmd_type = REQ_TYPE_SPECIAL;
980 rq->cmd_flags |= REQ_SOFTBARRIER;
981 1003
982 rq->special = data; 1004 rq->special = data;
983 1005
@@ -991,7 +1013,7 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
991 1013
992 drive_stat_acct(rq, 1); 1014 drive_stat_acct(rq, 1);
993 __elv_add_request(q, rq, where, 0); 1015 __elv_add_request(q, rq, where, 0);
994 blk_start_queueing(q); 1016 __blk_run_queue(q);
995 spin_unlock_irqrestore(q->queue_lock, flags); 1017 spin_unlock_irqrestore(q->queue_lock, flags);
996} 1018}
997EXPORT_SYMBOL(blk_insert_request); 1019EXPORT_SYMBOL(blk_insert_request);
@@ -1113,16 +1135,13 @@ void init_request_from_bio(struct request *req, struct bio *bio)
1113 if (bio_failfast_driver(bio)) 1135 if (bio_failfast_driver(bio))
1114 req->cmd_flags |= REQ_FAILFAST_DRIVER; 1136 req->cmd_flags |= REQ_FAILFAST_DRIVER;
1115 1137
1116 /*
1117 * REQ_BARRIER implies no merging, but lets make it explicit
1118 */
1119 if (unlikely(bio_discard(bio))) { 1138 if (unlikely(bio_discard(bio))) {
1120 req->cmd_flags |= REQ_DISCARD; 1139 req->cmd_flags |= REQ_DISCARD;
1121 if (bio_barrier(bio)) 1140 if (bio_barrier(bio))
1122 req->cmd_flags |= REQ_SOFTBARRIER; 1141 req->cmd_flags |= REQ_SOFTBARRIER;
1123 req->q->prepare_discard_fn(req->q, req); 1142 req->q->prepare_discard_fn(req->q, req);
1124 } else if (unlikely(bio_barrier(bio))) 1143 } else if (unlikely(bio_barrier(bio)))
1125 req->cmd_flags |= (REQ_HARDBARRIER | REQ_NOMERGE); 1144 req->cmd_flags |= REQ_HARDBARRIER;
1126 1145
1127 if (bio_sync(bio)) 1146 if (bio_sync(bio))
1128 req->cmd_flags |= REQ_RW_SYNC; 1147 req->cmd_flags |= REQ_RW_SYNC;
@@ -1132,9 +1151,8 @@ void init_request_from_bio(struct request *req, struct bio *bio)
1132 req->cmd_flags |= REQ_NOIDLE; 1151 req->cmd_flags |= REQ_NOIDLE;
1133 1152
1134 req->errors = 0; 1153 req->errors = 0;
1135 req->hard_sector = req->sector = bio->bi_sector; 1154 req->__sector = bio->bi_sector;
1136 req->ioprio = bio_prio(bio); 1155 req->ioprio = bio_prio(bio);
1137 req->start_time = jiffies;
1138 blk_rq_bio_prep(req->q, req, bio); 1156 blk_rq_bio_prep(req->q, req, bio);
1139} 1157}
1140 1158
@@ -1150,14 +1168,13 @@ static inline bool queue_should_plug(struct request_queue *q)
1150static int __make_request(struct request_queue *q, struct bio *bio) 1168static int __make_request(struct request_queue *q, struct bio *bio)
1151{ 1169{
1152 struct request *req; 1170 struct request *req;
1153 int el_ret, nr_sectors; 1171 int el_ret;
1172 unsigned int bytes = bio->bi_size;
1154 const unsigned short prio = bio_prio(bio); 1173 const unsigned short prio = bio_prio(bio);
1155 const int sync = bio_sync(bio); 1174 const int sync = bio_sync(bio);
1156 const int unplug = bio_unplug(bio); 1175 const int unplug = bio_unplug(bio);
1157 int rw_flags; 1176 int rw_flags;
1158 1177
1159 nr_sectors = bio_sectors(bio);
1160
1161 /* 1178 /*
1162 * low level driver can indicate that it wants pages above a 1179 * low level driver can indicate that it wants pages above a
1163 * certain limit bounced to low memory (ie for highmem, or even 1180 * certain limit bounced to low memory (ie for highmem, or even
@@ -1182,7 +1199,7 @@ static int __make_request(struct request_queue *q, struct bio *bio)
1182 1199
1183 req->biotail->bi_next = bio; 1200 req->biotail->bi_next = bio;
1184 req->biotail = bio; 1201 req->biotail = bio;
1185 req->nr_sectors = req->hard_nr_sectors += nr_sectors; 1202 req->__data_len += bytes;
1186 req->ioprio = ioprio_best(req->ioprio, prio); 1203 req->ioprio = ioprio_best(req->ioprio, prio);
1187 if (!blk_rq_cpu_valid(req)) 1204 if (!blk_rq_cpu_valid(req))
1188 req->cpu = bio->bi_comp_cpu; 1205 req->cpu = bio->bi_comp_cpu;
@@ -1208,10 +1225,8 @@ static int __make_request(struct request_queue *q, struct bio *bio)
1208 * not touch req->buffer either... 1225 * not touch req->buffer either...
1209 */ 1226 */
1210 req->buffer = bio_data(bio); 1227 req->buffer = bio_data(bio);
1211 req->current_nr_sectors = bio_cur_sectors(bio); 1228 req->__sector = bio->bi_sector;
1212 req->hard_cur_sectors = req->current_nr_sectors; 1229 req->__data_len += bytes;
1213 req->sector = req->hard_sector = bio->bi_sector;
1214 req->nr_sectors = req->hard_nr_sectors += nr_sectors;
1215 req->ioprio = ioprio_best(req->ioprio, prio); 1230 req->ioprio = ioprio_best(req->ioprio, prio);
1216 if (!blk_rq_cpu_valid(req)) 1231 if (!blk_rq_cpu_valid(req))
1217 req->cpu = bio->bi_comp_cpu; 1232 req->cpu = bio->bi_comp_cpu;
@@ -1593,8 +1608,8 @@ EXPORT_SYMBOL(submit_bio);
1593 */ 1608 */
1594int blk_rq_check_limits(struct request_queue *q, struct request *rq) 1609int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1595{ 1610{
1596 if (rq->nr_sectors > q->max_sectors || 1611 if (blk_rq_sectors(rq) > q->max_sectors ||
1597 rq->data_len > q->max_hw_sectors << 9) { 1612 blk_rq_bytes(rq) > q->max_hw_sectors << 9) {
1598 printk(KERN_ERR "%s: over max size limit.\n", __func__); 1613 printk(KERN_ERR "%s: over max size limit.\n", __func__);
1599 return -EIO; 1614 return -EIO;
1600 } 1615 }
@@ -1651,40 +1666,15 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1651} 1666}
1652EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 1667EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1653 1668
1654/**
1655 * blkdev_dequeue_request - dequeue request and start timeout timer
1656 * @req: request to dequeue
1657 *
1658 * Dequeue @req and start timeout timer on it. This hands off the
1659 * request to the driver.
1660 *
1661 * Block internal functions which don't want to start timer should
1662 * call elv_dequeue_request().
1663 */
1664void blkdev_dequeue_request(struct request *req)
1665{
1666 elv_dequeue_request(req->q, req);
1667
1668 /*
1669 * We are now handing the request to the hardware, add the
1670 * timeout handler.
1671 */
1672 blk_add_timer(req);
1673}
1674EXPORT_SYMBOL(blkdev_dequeue_request);
1675
1676static void blk_account_io_completion(struct request *req, unsigned int bytes) 1669static void blk_account_io_completion(struct request *req, unsigned int bytes)
1677{ 1670{
1678 if (!blk_do_io_stat(req)) 1671 if (blk_do_io_stat(req)) {
1679 return;
1680
1681 if (blk_fs_request(req)) {
1682 const int rw = rq_data_dir(req); 1672 const int rw = rq_data_dir(req);
1683 struct hd_struct *part; 1673 struct hd_struct *part;
1684 int cpu; 1674 int cpu;
1685 1675
1686 cpu = part_stat_lock(); 1676 cpu = part_stat_lock();
1687 part = disk_map_sector_rcu(req->rq_disk, req->sector); 1677 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
1688 part_stat_add(cpu, part, sectors[rw], bytes >> 9); 1678 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
1689 part_stat_unlock(); 1679 part_stat_unlock();
1690 } 1680 }
@@ -1692,22 +1682,19 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes)
1692 1682
1693static void blk_account_io_done(struct request *req) 1683static void blk_account_io_done(struct request *req)
1694{ 1684{
1695 if (!blk_do_io_stat(req))
1696 return;
1697
1698 /* 1685 /*
1699 * Account IO completion. bar_rq isn't accounted as a normal 1686 * Account IO completion. bar_rq isn't accounted as a normal
1700 * IO on queueing nor completion. Accounting the containing 1687 * IO on queueing nor completion. Accounting the containing
1701 * request is enough. 1688 * request is enough.
1702 */ 1689 */
1703 if (blk_fs_request(req) && req != &req->q->bar_rq) { 1690 if (blk_do_io_stat(req) && req != &req->q->bar_rq) {
1704 unsigned long duration = jiffies - req->start_time; 1691 unsigned long duration = jiffies - req->start_time;
1705 const int rw = rq_data_dir(req); 1692 const int rw = rq_data_dir(req);
1706 struct hd_struct *part; 1693 struct hd_struct *part;
1707 int cpu; 1694 int cpu;
1708 1695
1709 cpu = part_stat_lock(); 1696 cpu = part_stat_lock();
1710 part = disk_map_sector_rcu(req->rq_disk, req->sector); 1697 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
1711 1698
1712 part_stat_inc(cpu, part, ios[rw]); 1699 part_stat_inc(cpu, part, ios[rw]);
1713 part_stat_add(cpu, part, ticks[rw], duration); 1700 part_stat_add(cpu, part, ticks[rw], duration);
@@ -1719,38 +1706,218 @@ static void blk_account_io_done(struct request *req)
1719} 1706}
1720 1707
1721/** 1708/**
1722 * __end_that_request_first - end I/O on a request 1709 * blk_peek_request - peek at the top of a request queue
1723 * @req: the request being processed 1710 * @q: request queue to peek at
1711 *
1712 * Description:
1713 * Return the request at the top of @q. The returned request
1714 * should be started using blk_start_request() before LLD starts
1715 * processing it.
1716 *
1717 * Return:
1718 * Pointer to the request at the top of @q if available. Null
1719 * otherwise.
1720 *
1721 * Context:
1722 * queue_lock must be held.
1723 */
1724struct request *blk_peek_request(struct request_queue *q)
1725{
1726 struct request *rq;
1727 int ret;
1728
1729 while ((rq = __elv_next_request(q)) != NULL) {
1730 if (!(rq->cmd_flags & REQ_STARTED)) {
1731 /*
1732 * This is the first time the device driver
1733 * sees this request (possibly after
1734 * requeueing). Notify IO scheduler.
1735 */
1736 if (blk_sorted_rq(rq))
1737 elv_activate_rq(q, rq);
1738
1739 /*
1740 * just mark as started even if we don't start
1741 * it, a request that has been delayed should
1742 * not be passed by new incoming requests
1743 */
1744 rq->cmd_flags |= REQ_STARTED;
1745 trace_block_rq_issue(q, rq);
1746 }
1747
1748 if (!q->boundary_rq || q->boundary_rq == rq) {
1749 q->end_sector = rq_end_sector(rq);
1750 q->boundary_rq = NULL;
1751 }
1752
1753 if (rq->cmd_flags & REQ_DONTPREP)
1754 break;
1755
1756 if (q->dma_drain_size && blk_rq_bytes(rq)) {
1757 /*
1758 * make sure space for the drain appears we
1759 * know we can do this because max_hw_segments
1760 * has been adjusted to be one fewer than the
1761 * device can handle
1762 */
1763 rq->nr_phys_segments++;
1764 }
1765
1766 if (!q->prep_rq_fn)
1767 break;
1768
1769 ret = q->prep_rq_fn(q, rq);
1770 if (ret == BLKPREP_OK) {
1771 break;
1772 } else if (ret == BLKPREP_DEFER) {
1773 /*
1774 * the request may have been (partially) prepped.
1775 * we need to keep this request in the front to
1776 * avoid resource deadlock. REQ_STARTED will
1777 * prevent other fs requests from passing this one.
1778 */
1779 if (q->dma_drain_size && blk_rq_bytes(rq) &&
1780 !(rq->cmd_flags & REQ_DONTPREP)) {
1781 /*
1782 * remove the space for the drain we added
1783 * so that we don't add it again
1784 */
1785 --rq->nr_phys_segments;
1786 }
1787
1788 rq = NULL;
1789 break;
1790 } else if (ret == BLKPREP_KILL) {
1791 rq->cmd_flags |= REQ_QUIET;
1792 __blk_end_request_all(rq, -EIO);
1793 } else {
1794 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
1795 break;
1796 }
1797 }
1798
1799 return rq;
1800}
1801EXPORT_SYMBOL(blk_peek_request);
1802
1803void blk_dequeue_request(struct request *rq)
1804{
1805 struct request_queue *q = rq->q;
1806
1807 BUG_ON(list_empty(&rq->queuelist));
1808 BUG_ON(ELV_ON_HASH(rq));
1809
1810 list_del_init(&rq->queuelist);
1811
1812 /*
1813 * the time frame between a request being removed from the lists
1814 * and to it is freed is accounted as io that is in progress at
1815 * the driver side.
1816 */
1817 if (blk_account_rq(rq))
1818 q->in_flight[rq_is_sync(rq)]++;
1819}
1820
1821/**
1822 * blk_start_request - start request processing on the driver
1823 * @req: request to dequeue
1824 *
1825 * Description:
1826 * Dequeue @req and start timeout timer on it. This hands off the
1827 * request to the driver.
1828 *
1829 * Block internal functions which don't want to start timer should
1830 * call blk_dequeue_request().
1831 *
1832 * Context:
1833 * queue_lock must be held.
1834 */
1835void blk_start_request(struct request *req)
1836{
1837 blk_dequeue_request(req);
1838
1839 /*
1840 * We are now handing the request to the hardware, initialize
1841 * resid_len to full count and add the timeout handler.
1842 */
1843 req->resid_len = blk_rq_bytes(req);
1844 blk_add_timer(req);
1845}
1846EXPORT_SYMBOL(blk_start_request);
1847
1848/**
1849 * blk_fetch_request - fetch a request from a request queue
1850 * @q: request queue to fetch a request from
1851 *
1852 * Description:
1853 * Return the request at the top of @q. The request is started on
1854 * return and LLD can start processing it immediately.
1855 *
1856 * Return:
1857 * Pointer to the request at the top of @q if available. Null
1858 * otherwise.
1859 *
1860 * Context:
1861 * queue_lock must be held.
1862 */
1863struct request *blk_fetch_request(struct request_queue *q)
1864{
1865 struct request *rq;
1866
1867 rq = blk_peek_request(q);
1868 if (rq)
1869 blk_start_request(rq);
1870 return rq;
1871}
1872EXPORT_SYMBOL(blk_fetch_request);
1873
1874/**
1875 * blk_update_request - Special helper function for request stacking drivers
1876 * @rq: the request being processed
1724 * @error: %0 for success, < %0 for error 1877 * @error: %0 for success, < %0 for error
1725 * @nr_bytes: number of bytes to complete 1878 * @nr_bytes: number of bytes to complete @rq
1726 * 1879 *
1727 * Description: 1880 * Description:
1728 * Ends I/O on a number of bytes attached to @req, and sets it up 1881 * Ends I/O on a number of bytes attached to @rq, but doesn't complete
1729 * for the next range of segments (if any) in the cluster. 1882 * the request structure even if @rq doesn't have leftover.
1883 * If @rq has leftover, sets it up for the next range of segments.
1884 *
1885 * This special helper function is only for request stacking drivers
1886 * (e.g. request-based dm) so that they can handle partial completion.
1887 * Actual device drivers should use blk_end_request instead.
1888 *
1889 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
1890 * %false return from this function.
1730 * 1891 *
1731 * Return: 1892 * Return:
1732 * %0 - we are done with this request, call end_that_request_last() 1893 * %false - this request doesn't have any more data
1733 * %1 - still buffers pending for this request 1894 * %true - this request has more data
1734 **/ 1895 **/
1735static int __end_that_request_first(struct request *req, int error, 1896bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
1736 int nr_bytes)
1737{ 1897{
1738 int total_bytes, bio_nbytes, next_idx = 0; 1898 int total_bytes, bio_nbytes, next_idx = 0;
1739 struct bio *bio; 1899 struct bio *bio;
1740 1900
1901 if (!req->bio)
1902 return false;
1903
1741 trace_block_rq_complete(req->q, req); 1904 trace_block_rq_complete(req->q, req);
1742 1905
1743 /* 1906 /*
1744 * for a REQ_TYPE_BLOCK_PC request, we want to carry any eventual 1907 * For fs requests, rq is just carrier of independent bio's
1745 * sense key with us all the way through 1908 * and each partial completion should be handled separately.
1909 * Reset per-request error on each partial completion.
1910 *
1911 * TODO: tj: This is too subtle. It would be better to let
1912 * low level drivers do what they see fit.
1746 */ 1913 */
1747 if (!blk_pc_request(req)) 1914 if (blk_fs_request(req))
1748 req->errors = 0; 1915 req->errors = 0;
1749 1916
1750 if (error && (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))) { 1917 if (error && (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))) {
1751 printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n", 1918 printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n",
1752 req->rq_disk ? req->rq_disk->disk_name : "?", 1919 req->rq_disk ? req->rq_disk->disk_name : "?",
1753 (unsigned long long)req->sector); 1920 (unsigned long long)blk_rq_pos(req));
1754 } 1921 }
1755 1922
1756 blk_account_io_completion(req, nr_bytes); 1923 blk_account_io_completion(req, nr_bytes);
@@ -1810,8 +1977,15 @@ static int __end_that_request_first(struct request *req, int error,
1810 /* 1977 /*
1811 * completely done 1978 * completely done
1812 */ 1979 */
1813 if (!req->bio) 1980 if (!req->bio) {
1814 return 0; 1981 /*
1982 * Reset counters so that the request stacking driver
1983 * can find how many bytes remain in the request
1984 * later.
1985 */
1986 req->__data_len = 0;
1987 return false;
1988 }
1815 1989
1816 /* 1990 /*
1817 * if the request wasn't completed, update state 1991 * if the request wasn't completed, update state
@@ -1823,22 +1997,56 @@ static int __end_that_request_first(struct request *req, int error,
1823 bio_iovec(bio)->bv_len -= nr_bytes; 1997 bio_iovec(bio)->bv_len -= nr_bytes;
1824 } 1998 }
1825 1999
1826 blk_recalc_rq_sectors(req, total_bytes >> 9); 2000 req->__data_len -= total_bytes;
2001 req->buffer = bio_data(req->bio);
2002
2003 /* update sector only for requests with clear definition of sector */
2004 if (blk_fs_request(req) || blk_discard_rq(req))
2005 req->__sector += total_bytes >> 9;
2006
2007 /*
2008 * If total number of sectors is less than the first segment
2009 * size, something has gone terribly wrong.
2010 */
2011 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2012 printk(KERN_ERR "blk: request botched\n");
2013 req->__data_len = blk_rq_cur_bytes(req);
2014 }
2015
2016 /* recalculate the number of segments */
1827 blk_recalc_rq_segments(req); 2017 blk_recalc_rq_segments(req);
1828 return 1; 2018
2019 return true;
2020}
2021EXPORT_SYMBOL_GPL(blk_update_request);
2022
2023static bool blk_update_bidi_request(struct request *rq, int error,
2024 unsigned int nr_bytes,
2025 unsigned int bidi_bytes)
2026{
2027 if (blk_update_request(rq, error, nr_bytes))
2028 return true;
2029
2030 /* Bidi request must be completed as a whole */
2031 if (unlikely(blk_bidi_rq(rq)) &&
2032 blk_update_request(rq->next_rq, error, bidi_bytes))
2033 return true;
2034
2035 add_disk_randomness(rq->rq_disk);
2036
2037 return false;
1829} 2038}
1830 2039
1831/* 2040/*
1832 * queue lock must be held 2041 * queue lock must be held
1833 */ 2042 */
1834static void end_that_request_last(struct request *req, int error) 2043static void blk_finish_request(struct request *req, int error)
1835{ 2044{
2045 BUG_ON(blk_queued_rq(req));
2046
1836 if (blk_rq_tagged(req)) 2047 if (blk_rq_tagged(req))
1837 blk_queue_end_tag(req->q, req); 2048 blk_queue_end_tag(req->q, req);
1838 2049
1839 if (blk_queued_rq(req))
1840 elv_dequeue_request(req->q, req);
1841
1842 if (unlikely(laptop_mode) && blk_fs_request(req)) 2050 if (unlikely(laptop_mode) && blk_fs_request(req))
1843 laptop_io_completion(); 2051 laptop_io_completion();
1844 2052
@@ -1857,117 +2065,62 @@ static void end_that_request_last(struct request *req, int error)
1857} 2065}
1858 2066
1859/** 2067/**
1860 * blk_rq_bytes - Returns bytes left to complete in the entire request 2068 * blk_end_bidi_request - Complete a bidi request
1861 * @rq: the request being processed 2069 * @rq: the request to complete
1862 **/ 2070 * @error: %0 for success, < %0 for error
1863unsigned int blk_rq_bytes(struct request *rq) 2071 * @nr_bytes: number of bytes to complete @rq
1864{ 2072 * @bidi_bytes: number of bytes to complete @rq->next_rq
1865 if (blk_fs_request(rq))
1866 return rq->hard_nr_sectors << 9;
1867
1868 return rq->data_len;
1869}
1870EXPORT_SYMBOL_GPL(blk_rq_bytes);
1871
1872/**
1873 * blk_rq_cur_bytes - Returns bytes left to complete in the current segment
1874 * @rq: the request being processed
1875 **/
1876unsigned int blk_rq_cur_bytes(struct request *rq)
1877{
1878 if (blk_fs_request(rq))
1879 return rq->current_nr_sectors << 9;
1880
1881 if (rq->bio)
1882 return rq->bio->bi_size;
1883
1884 return rq->data_len;
1885}
1886EXPORT_SYMBOL_GPL(blk_rq_cur_bytes);
1887
1888/**
1889 * end_request - end I/O on the current segment of the request
1890 * @req: the request being processed
1891 * @uptodate: error value or %0/%1 uptodate flag
1892 * 2073 *
1893 * Description: 2074 * Description:
1894 * Ends I/O on the current segment of a request. If that is the only 2075 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
1895 * remaining segment, the request is also completed and freed. 2076 * Drivers that supports bidi can safely call this member for any
1896 * 2077 * type of request, bidi or uni. In the later case @bidi_bytes is
1897 * This is a remnant of how older block drivers handled I/O completions. 2078 * just ignored.
1898 * Modern drivers typically end I/O on the full request in one go, unless 2079 *
1899 * they have a residual value to account for. For that case this function 2080 * Return:
1900 * isn't really useful, unless the residual just happens to be the 2081 * %false - we are done with this request
1901 * full current segment. In other words, don't use this function in new 2082 * %true - still buffers pending for this request
1902 * code. Use blk_end_request() or __blk_end_request() to end a request.
1903 **/ 2083 **/
1904void end_request(struct request *req, int uptodate) 2084static bool blk_end_bidi_request(struct request *rq, int error,
1905{
1906 int error = 0;
1907
1908 if (uptodate <= 0)
1909 error = uptodate ? uptodate : -EIO;
1910
1911 __blk_end_request(req, error, req->hard_cur_sectors << 9);
1912}
1913EXPORT_SYMBOL(end_request);
1914
1915static int end_that_request_data(struct request *rq, int error,
1916 unsigned int nr_bytes, unsigned int bidi_bytes) 2085 unsigned int nr_bytes, unsigned int bidi_bytes)
1917{ 2086{
1918 if (rq->bio) { 2087 struct request_queue *q = rq->q;
1919 if (__end_that_request_first(rq, error, nr_bytes)) 2088 unsigned long flags;
1920 return 1;
1921 2089
1922 /* Bidi request must be completed as a whole */ 2090 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
1923 if (blk_bidi_rq(rq) && 2091 return true;
1924 __end_that_request_first(rq->next_rq, error, bidi_bytes))
1925 return 1;
1926 }
1927 2092
1928 return 0; 2093 spin_lock_irqsave(q->queue_lock, flags);
2094 blk_finish_request(rq, error);
2095 spin_unlock_irqrestore(q->queue_lock, flags);
2096
2097 return false;
1929} 2098}
1930 2099
1931/** 2100/**
1932 * blk_end_io - Generic end_io function to complete a request. 2101 * __blk_end_bidi_request - Complete a bidi request with queue lock held
1933 * @rq: the request being processed 2102 * @rq: the request to complete
1934 * @error: %0 for success, < %0 for error 2103 * @error: %0 for success, < %0 for error
1935 * @nr_bytes: number of bytes to complete @rq 2104 * @nr_bytes: number of bytes to complete @rq
1936 * @bidi_bytes: number of bytes to complete @rq->next_rq 2105 * @bidi_bytes: number of bytes to complete @rq->next_rq
1937 * @drv_callback: function called between completion of bios in the request
1938 * and completion of the request.
1939 * If the callback returns non %0, this helper returns without
1940 * completion of the request.
1941 * 2106 *
1942 * Description: 2107 * Description:
1943 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. 2108 * Identical to blk_end_bidi_request() except that queue lock is
1944 * If @rq has leftover, sets it up for the next range of segments. 2109 * assumed to be locked on entry and remains so on return.
1945 * 2110 *
1946 * Return: 2111 * Return:
1947 * %0 - we are done with this request 2112 * %false - we are done with this request
1948 * %1 - this request is not freed yet, it still has pending buffers. 2113 * %true - still buffers pending for this request
1949 **/ 2114 **/
1950static int blk_end_io(struct request *rq, int error, unsigned int nr_bytes, 2115static bool __blk_end_bidi_request(struct request *rq, int error,
1951 unsigned int bidi_bytes, 2116 unsigned int nr_bytes, unsigned int bidi_bytes)
1952 int (drv_callback)(struct request *))
1953{ 2117{
1954 struct request_queue *q = rq->q; 2118 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
1955 unsigned long flags = 0UL; 2119 return true;
1956
1957 if (end_that_request_data(rq, error, nr_bytes, bidi_bytes))
1958 return 1;
1959
1960 /* Special feature for tricky drivers */
1961 if (drv_callback && drv_callback(rq))
1962 return 1;
1963
1964 add_disk_randomness(rq->rq_disk);
1965 2120
1966 spin_lock_irqsave(q->queue_lock, flags); 2121 blk_finish_request(rq, error);
1967 end_that_request_last(rq, error);
1968 spin_unlock_irqrestore(q->queue_lock, flags);
1969 2122
1970 return 0; 2123 return false;
1971} 2124}
1972 2125
1973/** 2126/**
@@ -1981,124 +2134,112 @@ static int blk_end_io(struct request *rq, int error, unsigned int nr_bytes,
1981 * If @rq has leftover, sets it up for the next range of segments. 2134 * If @rq has leftover, sets it up for the next range of segments.
1982 * 2135 *
1983 * Return: 2136 * Return:
1984 * %0 - we are done with this request 2137 * %false - we are done with this request
1985 * %1 - still buffers pending for this request 2138 * %true - still buffers pending for this request
1986 **/ 2139 **/
1987int blk_end_request(struct request *rq, int error, unsigned int nr_bytes) 2140bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
1988{ 2141{
1989 return blk_end_io(rq, error, nr_bytes, 0, NULL); 2142 return blk_end_bidi_request(rq, error, nr_bytes, 0);
1990} 2143}
1991EXPORT_SYMBOL_GPL(blk_end_request); 2144EXPORT_SYMBOL_GPL(blk_end_request);
1992 2145
1993/** 2146/**
1994 * __blk_end_request - Helper function for drivers to complete the request. 2147 * blk_end_request_all - Helper function for drives to finish the request.
1995 * @rq: the request being processed 2148 * @rq: the request to finish
1996 * @error: %0 for success, < %0 for error 2149 * @err: %0 for success, < %0 for error
1997 * @nr_bytes: number of bytes to complete
1998 * 2150 *
1999 * Description: 2151 * Description:
2000 * Must be called with queue lock held unlike blk_end_request(). 2152 * Completely finish @rq.
2001 * 2153 */
2002 * Return: 2154void blk_end_request_all(struct request *rq, int error)
2003 * %0 - we are done with this request
2004 * %1 - still buffers pending for this request
2005 **/
2006int __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2007{ 2155{
2008 if (rq->bio && __end_that_request_first(rq, error, nr_bytes)) 2156 bool pending;
2009 return 1; 2157 unsigned int bidi_bytes = 0;
2010 2158
2011 add_disk_randomness(rq->rq_disk); 2159 if (unlikely(blk_bidi_rq(rq)))
2160 bidi_bytes = blk_rq_bytes(rq->next_rq);
2012 2161
2013 end_that_request_last(rq, error); 2162 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2163 BUG_ON(pending);
2164}
2165EXPORT_SYMBOL_GPL(blk_end_request_all);
2014 2166
2015 return 0; 2167/**
2168 * blk_end_request_cur - Helper function to finish the current request chunk.
2169 * @rq: the request to finish the current chunk for
2170 * @err: %0 for success, < %0 for error
2171 *
2172 * Description:
2173 * Complete the current consecutively mapped chunk from @rq.
2174 *
2175 * Return:
2176 * %false - we are done with this request
2177 * %true - still buffers pending for this request
2178 */
2179bool blk_end_request_cur(struct request *rq, int error)
2180{
2181 return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2016} 2182}
2017EXPORT_SYMBOL_GPL(__blk_end_request); 2183EXPORT_SYMBOL_GPL(blk_end_request_cur);
2018 2184
2019/** 2185/**
2020 * blk_end_bidi_request - Helper function for drivers to complete bidi request. 2186 * __blk_end_request - Helper function for drivers to complete the request.
2021 * @rq: the bidi request being processed 2187 * @rq: the request being processed
2022 * @error: %0 for success, < %0 for error 2188 * @error: %0 for success, < %0 for error
2023 * @nr_bytes: number of bytes to complete @rq 2189 * @nr_bytes: number of bytes to complete
2024 * @bidi_bytes: number of bytes to complete @rq->next_rq
2025 * 2190 *
2026 * Description: 2191 * Description:
2027 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. 2192 * Must be called with queue lock held unlike blk_end_request().
2028 * 2193 *
2029 * Return: 2194 * Return:
2030 * %0 - we are done with this request 2195 * %false - we are done with this request
2031 * %1 - still buffers pending for this request 2196 * %true - still buffers pending for this request
2032 **/ 2197 **/
2033int blk_end_bidi_request(struct request *rq, int error, unsigned int nr_bytes, 2198bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2034 unsigned int bidi_bytes)
2035{ 2199{
2036 return blk_end_io(rq, error, nr_bytes, bidi_bytes, NULL); 2200 return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2037} 2201}
2038EXPORT_SYMBOL_GPL(blk_end_bidi_request); 2202EXPORT_SYMBOL_GPL(__blk_end_request);
2039 2203
2040/** 2204/**
2041 * blk_update_request - Special helper function for request stacking drivers 2205 * __blk_end_request_all - Helper function for drives to finish the request.
2042 * @rq: the request being processed 2206 * @rq: the request to finish
2043 * @error: %0 for success, < %0 for error 2207 * @err: %0 for success, < %0 for error
2044 * @nr_bytes: number of bytes to complete @rq
2045 * 2208 *
2046 * Description: 2209 * Description:
2047 * Ends I/O on a number of bytes attached to @rq, but doesn't complete 2210 * Completely finish @rq. Must be called with queue lock held.
2048 * the request structure even if @rq doesn't have leftover.
2049 * If @rq has leftover, sets it up for the next range of segments.
2050 *
2051 * This special helper function is only for request stacking drivers
2052 * (e.g. request-based dm) so that they can handle partial completion.
2053 * Actual device drivers should use blk_end_request instead.
2054 */ 2211 */
2055void blk_update_request(struct request *rq, int error, unsigned int nr_bytes) 2212void __blk_end_request_all(struct request *rq, int error)
2056{ 2213{
2057 if (!end_that_request_data(rq, error, nr_bytes, 0)) { 2214 bool pending;
2058 /* 2215 unsigned int bidi_bytes = 0;
2059 * These members are not updated in end_that_request_data() 2216
2060 * when all bios are completed. 2217 if (unlikely(blk_bidi_rq(rq)))
2061 * Update them so that the request stacking driver can find 2218 bidi_bytes = blk_rq_bytes(rq->next_rq);
2062 * how many bytes remain in the request later. 2219
2063 */ 2220 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2064 rq->nr_sectors = rq->hard_nr_sectors = 0; 2221 BUG_ON(pending);
2065 rq->current_nr_sectors = rq->hard_cur_sectors = 0;
2066 }
2067} 2222}
2068EXPORT_SYMBOL_GPL(blk_update_request); 2223EXPORT_SYMBOL_GPL(__blk_end_request_all);
2069 2224
2070/** 2225/**
2071 * blk_end_request_callback - Special helper function for tricky drivers 2226 * __blk_end_request_cur - Helper function to finish the current request chunk.
2072 * @rq: the request being processed 2227 * @rq: the request to finish the current chunk for
2073 * @error: %0 for success, < %0 for error 2228 * @err: %0 for success, < %0 for error
2074 * @nr_bytes: number of bytes to complete
2075 * @drv_callback: function called between completion of bios in the request
2076 * and completion of the request.
2077 * If the callback returns non %0, this helper returns without
2078 * completion of the request.
2079 * 2229 *
2080 * Description: 2230 * Description:
2081 * Ends I/O on a number of bytes attached to @rq. 2231 * Complete the current consecutively mapped chunk from @rq. Must
2082 * If @rq has leftover, sets it up for the next range of segments. 2232 * be called with queue lock held.
2083 *
2084 * This special helper function is used only for existing tricky drivers.
2085 * (e.g. cdrom_newpc_intr() of ide-cd)
2086 * This interface will be removed when such drivers are rewritten.
2087 * Don't use this interface in other places anymore.
2088 * 2233 *
2089 * Return: 2234 * Return:
2090 * %0 - we are done with this request 2235 * %false - we are done with this request
2091 * %1 - this request is not freed yet. 2236 * %true - still buffers pending for this request
2092 * this request still has pending buffers or 2237 */
2093 * the driver doesn't want to finish this request yet. 2238bool __blk_end_request_cur(struct request *rq, int error)
2094 **/
2095int blk_end_request_callback(struct request *rq, int error,
2096 unsigned int nr_bytes,
2097 int (drv_callback)(struct request *))
2098{ 2239{
2099 return blk_end_io(rq, error, nr_bytes, 0, drv_callback); 2240 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2100} 2241}
2101EXPORT_SYMBOL_GPL(blk_end_request_callback); 2242EXPORT_SYMBOL_GPL(__blk_end_request_cur);
2102 2243
2103void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 2244void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2104 struct bio *bio) 2245 struct bio *bio)
@@ -2111,11 +2252,7 @@ void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2111 rq->nr_phys_segments = bio_phys_segments(q, bio); 2252 rq->nr_phys_segments = bio_phys_segments(q, bio);
2112 rq->buffer = bio_data(bio); 2253 rq->buffer = bio_data(bio);
2113 } 2254 }
2114 rq->current_nr_sectors = bio_cur_sectors(bio); 2255 rq->__data_len = bio->bi_size;
2115 rq->hard_cur_sectors = rq->current_nr_sectors;
2116 rq->hard_nr_sectors = rq->nr_sectors = bio_sectors(bio);
2117 rq->data_len = bio->bi_size;
2118
2119 rq->bio = rq->biotail = bio; 2256 rq->bio = rq->biotail = bio;
2120 2257
2121 if (bio->bi_bdev) 2258 if (bio->bi_bdev)
@@ -2158,6 +2295,9 @@ EXPORT_SYMBOL(kblockd_schedule_work);
2158 2295
2159int __init blk_dev_init(void) 2296int __init blk_dev_init(void)
2160{ 2297{
2298 BUILD_BUG_ON(__REQ_NR_BITS > 8 *
2299 sizeof(((struct request *)0)->cmd_flags));
2300
2161 kblockd_workqueue = create_workqueue("kblockd"); 2301 kblockd_workqueue = create_workqueue("kblockd");
2162 if (!kblockd_workqueue) 2302 if (!kblockd_workqueue)
2163 panic("Failed to create kblockd\n"); 2303 panic("Failed to create kblockd\n");
diff --git a/block/blk-exec.c b/block/blk-exec.c
index 6af716d1e54e..49557e91f0da 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -51,7 +51,6 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
51 int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK; 51 int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
52 52
53 rq->rq_disk = bd_disk; 53 rq->rq_disk = bd_disk;
54 rq->cmd_flags |= REQ_NOMERGE;
55 rq->end_io = done; 54 rq->end_io = done;
56 WARN_ON(irqs_disabled()); 55 WARN_ON(irqs_disabled());
57 spin_lock_irq(q->queue_lock); 56 spin_lock_irq(q->queue_lock);
diff --git a/block/blk-map.c b/block/blk-map.c
index f103729b462f..ef2492adca7e 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -20,11 +20,10 @@ int blk_rq_append_bio(struct request_queue *q, struct request *rq,
20 rq->biotail->bi_next = bio; 20 rq->biotail->bi_next = bio;
21 rq->biotail = bio; 21 rq->biotail = bio;
22 22
23 rq->data_len += bio->bi_size; 23 rq->__data_len += bio->bi_size;
24 } 24 }
25 return 0; 25 return 0;
26} 26}
27EXPORT_SYMBOL(blk_rq_append_bio);
28 27
29static int __blk_rq_unmap_user(struct bio *bio) 28static int __blk_rq_unmap_user(struct bio *bio)
30{ 29{
@@ -156,7 +155,7 @@ int blk_rq_map_user(struct request_queue *q, struct request *rq,
156 if (!bio_flagged(bio, BIO_USER_MAPPED)) 155 if (!bio_flagged(bio, BIO_USER_MAPPED))
157 rq->cmd_flags |= REQ_COPY_USER; 156 rq->cmd_flags |= REQ_COPY_USER;
158 157
159 rq->buffer = rq->data = NULL; 158 rq->buffer = NULL;
160 return 0; 159 return 0;
161unmap_rq: 160unmap_rq:
162 blk_rq_unmap_user(bio); 161 blk_rq_unmap_user(bio);
@@ -235,7 +234,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
235 blk_queue_bounce(q, &bio); 234 blk_queue_bounce(q, &bio);
236 bio_get(bio); 235 bio_get(bio);
237 blk_rq_bio_prep(q, rq, bio); 236 blk_rq_bio_prep(q, rq, bio);
238 rq->buffer = rq->data = NULL; 237 rq->buffer = NULL;
239 return 0; 238 return 0;
240} 239}
241EXPORT_SYMBOL(blk_rq_map_user_iov); 240EXPORT_SYMBOL(blk_rq_map_user_iov);
@@ -282,7 +281,8 @@ EXPORT_SYMBOL(blk_rq_unmap_user);
282 * 281 *
283 * Description: 282 * Description:
284 * Data will be mapped directly if possible. Otherwise a bounce 283 * Data will be mapped directly if possible. Otherwise a bounce
285 * buffer is used. 284 * buffer is used. Can be called multple times to append multple
285 * buffers.
286 */ 286 */
287int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 287int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
288 unsigned int len, gfp_t gfp_mask) 288 unsigned int len, gfp_t gfp_mask)
@@ -290,6 +290,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
290 int reading = rq_data_dir(rq) == READ; 290 int reading = rq_data_dir(rq) == READ;
291 int do_copy = 0; 291 int do_copy = 0;
292 struct bio *bio; 292 struct bio *bio;
293 int ret;
293 294
294 if (len > (q->max_hw_sectors << 9)) 295 if (len > (q->max_hw_sectors << 9))
295 return -EINVAL; 296 return -EINVAL;
@@ -311,9 +312,15 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
311 if (do_copy) 312 if (do_copy)
312 rq->cmd_flags |= REQ_COPY_USER; 313 rq->cmd_flags |= REQ_COPY_USER;
313 314
314 blk_rq_bio_prep(q, rq, bio); 315 ret = blk_rq_append_bio(q, rq, bio);
316 if (unlikely(ret)) {
317 /* request is too big */
318 bio_put(bio);
319 return ret;
320 }
321
315 blk_queue_bounce(q, &rq->bio); 322 blk_queue_bounce(q, &rq->bio);
316 rq->buffer = rq->data = NULL; 323 rq->buffer = NULL;
317 return 0; 324 return 0;
318} 325}
319EXPORT_SYMBOL(blk_rq_map_kern); 326EXPORT_SYMBOL(blk_rq_map_kern);
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 23d2a6fe34a3..4974dd5767e5 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -9,35 +9,6 @@
9 9
10#include "blk.h" 10#include "blk.h"
11 11
12void blk_recalc_rq_sectors(struct request *rq, int nsect)
13{
14 if (blk_fs_request(rq) || blk_discard_rq(rq)) {
15 rq->hard_sector += nsect;
16 rq->hard_nr_sectors -= nsect;
17
18 /*
19 * Move the I/O submission pointers ahead if required.
20 */
21 if ((rq->nr_sectors >= rq->hard_nr_sectors) &&
22 (rq->sector <= rq->hard_sector)) {
23 rq->sector = rq->hard_sector;
24 rq->nr_sectors = rq->hard_nr_sectors;
25 rq->hard_cur_sectors = bio_cur_sectors(rq->bio);
26 rq->current_nr_sectors = rq->hard_cur_sectors;
27 rq->buffer = bio_data(rq->bio);
28 }
29
30 /*
31 * if total number of sectors is less than the first segment
32 * size, something has gone terribly wrong
33 */
34 if (rq->nr_sectors < rq->current_nr_sectors) {
35 printk(KERN_ERR "blk: request botched\n");
36 rq->nr_sectors = rq->current_nr_sectors;
37 }
38 }
39}
40
41static unsigned int __blk_recalc_rq_segments(struct request_queue *q, 12static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
42 struct bio *bio) 13 struct bio *bio)
43{ 14{
@@ -199,8 +170,9 @@ new_segment:
199 170
200 171
201 if (unlikely(rq->cmd_flags & REQ_COPY_USER) && 172 if (unlikely(rq->cmd_flags & REQ_COPY_USER) &&
202 (rq->data_len & q->dma_pad_mask)) { 173 (blk_rq_bytes(rq) & q->dma_pad_mask)) {
203 unsigned int pad_len = (q->dma_pad_mask & ~rq->data_len) + 1; 174 unsigned int pad_len =
175 (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
204 176
205 sg->length += pad_len; 177 sg->length += pad_len;
206 rq->extra_len += pad_len; 178 rq->extra_len += pad_len;
@@ -259,7 +231,7 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req,
259 else 231 else
260 max_sectors = q->max_sectors; 232 max_sectors = q->max_sectors;
261 233
262 if (req->nr_sectors + bio_sectors(bio) > max_sectors) { 234 if (blk_rq_sectors(req) + bio_sectors(bio) > max_sectors) {
263 req->cmd_flags |= REQ_NOMERGE; 235 req->cmd_flags |= REQ_NOMERGE;
264 if (req == q->last_merge) 236 if (req == q->last_merge)
265 q->last_merge = NULL; 237 q->last_merge = NULL;
@@ -284,7 +256,7 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req,
284 max_sectors = q->max_sectors; 256 max_sectors = q->max_sectors;
285 257
286 258
287 if (req->nr_sectors + bio_sectors(bio) > max_sectors) { 259 if (blk_rq_sectors(req) + bio_sectors(bio) > max_sectors) {
288 req->cmd_flags |= REQ_NOMERGE; 260 req->cmd_flags |= REQ_NOMERGE;
289 if (req == q->last_merge) 261 if (req == q->last_merge)
290 q->last_merge = NULL; 262 q->last_merge = NULL;
@@ -315,7 +287,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
315 /* 287 /*
316 * Will it become too large? 288 * Will it become too large?
317 */ 289 */
318 if ((req->nr_sectors + next->nr_sectors) > q->max_sectors) 290 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > q->max_sectors)
319 return 0; 291 return 0;
320 292
321 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; 293 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
@@ -345,7 +317,7 @@ static void blk_account_io_merge(struct request *req)
345 int cpu; 317 int cpu;
346 318
347 cpu = part_stat_lock(); 319 cpu = part_stat_lock();
348 part = disk_map_sector_rcu(req->rq_disk, req->sector); 320 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
349 321
350 part_round_stats(cpu, part); 322 part_round_stats(cpu, part);
351 part_dec_in_flight(part); 323 part_dec_in_flight(part);
@@ -366,7 +338,7 @@ static int attempt_merge(struct request_queue *q, struct request *req,
366 /* 338 /*
367 * not contiguous 339 * not contiguous
368 */ 340 */
369 if (req->sector + req->nr_sectors != next->sector) 341 if (blk_rq_pos(req) + blk_rq_sectors(req) != blk_rq_pos(next))
370 return 0; 342 return 0;
371 343
372 if (rq_data_dir(req) != rq_data_dir(next) 344 if (rq_data_dir(req) != rq_data_dir(next)
@@ -398,7 +370,7 @@ static int attempt_merge(struct request_queue *q, struct request *req,
398 req->biotail->bi_next = next->bio; 370 req->biotail->bi_next = next->bio;
399 req->biotail = next->biotail; 371 req->biotail = next->biotail;
400 372
401 req->nr_sectors = req->hard_nr_sectors += next->hard_nr_sectors; 373 req->__data_len += blk_rq_bytes(next);
402 374
403 elv_merge_requests(q, req, next); 375 elv_merge_requests(q, req, next);
404 376
diff --git a/block/blk-tag.c b/block/blk-tag.c
index 3c518e3303ae..2e5cfeb59333 100644
--- a/block/blk-tag.c
+++ b/block/blk-tag.c
@@ -336,7 +336,7 @@ EXPORT_SYMBOL(blk_queue_end_tag);
336int blk_queue_start_tag(struct request_queue *q, struct request *rq) 336int blk_queue_start_tag(struct request_queue *q, struct request *rq)
337{ 337{
338 struct blk_queue_tag *bqt = q->queue_tags; 338 struct blk_queue_tag *bqt = q->queue_tags;
339 unsigned max_depth, offset; 339 unsigned max_depth;
340 int tag; 340 int tag;
341 341
342 if (unlikely((rq->cmd_flags & REQ_QUEUED))) { 342 if (unlikely((rq->cmd_flags & REQ_QUEUED))) {
@@ -355,13 +355,16 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
355 * to starve sync IO on behalf of flooding async IO. 355 * to starve sync IO on behalf of flooding async IO.
356 */ 356 */
357 max_depth = bqt->max_depth; 357 max_depth = bqt->max_depth;
358 if (rq_is_sync(rq)) 358 if (!rq_is_sync(rq) && max_depth > 1) {
359 offset = 0; 359 max_depth -= 2;
360 else 360 if (!max_depth)
361 offset = max_depth >> 2; 361 max_depth = 1;
362 if (q->in_flight[0] > max_depth)
363 return 1;
364 }
362 365
363 do { 366 do {
364 tag = find_next_zero_bit(bqt->tag_map, max_depth, offset); 367 tag = find_first_zero_bit(bqt->tag_map, max_depth);
365 if (tag >= max_depth) 368 if (tag >= max_depth)
366 return 1; 369 return 1;
367 370
@@ -374,7 +377,7 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
374 rq->cmd_flags |= REQ_QUEUED; 377 rq->cmd_flags |= REQ_QUEUED;
375 rq->tag = tag; 378 rq->tag = tag;
376 bqt->tag_index[tag] = rq; 379 bqt->tag_index[tag] = rq;
377 blkdev_dequeue_request(rq); 380 blk_start_request(rq);
378 list_add(&rq->queuelist, &q->tag_busy_list); 381 list_add(&rq->queuelist, &q->tag_busy_list);
379 return 0; 382 return 0;
380} 383}
diff --git a/block/blk-timeout.c b/block/blk-timeout.c
index 1ec0d503cacd..1ba7e0aca878 100644
--- a/block/blk-timeout.c
+++ b/block/blk-timeout.c
@@ -122,10 +122,8 @@ void blk_rq_timed_out_timer(unsigned long data)
122 if (blk_mark_rq_complete(rq)) 122 if (blk_mark_rq_complete(rq))
123 continue; 123 continue;
124 blk_rq_timed_out(rq); 124 blk_rq_timed_out(rq);
125 } else { 125 } else if (!next || time_after(next, rq->deadline))
126 if (!next || time_after(next, rq->deadline)) 126 next = rq->deadline;
127 next = rq->deadline;
128 }
129 } 127 }
130 128
131 /* 129 /*
@@ -176,16 +174,14 @@ void blk_add_timer(struct request *req)
176 BUG_ON(!list_empty(&req->timeout_list)); 174 BUG_ON(!list_empty(&req->timeout_list));
177 BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags)); 175 BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));
178 176
179 if (req->timeout) 177 /*
180 req->deadline = jiffies + req->timeout; 178 * Some LLDs, like scsi, peek at the timeout to prevent a
181 else { 179 * command from being retried forever.
182 req->deadline = jiffies + q->rq_timeout; 180 */
183 /* 181 if (!req->timeout)
184 * Some LLDs, like scsi, peek at the timeout to prevent
185 * a command from being retried forever.
186 */
187 req->timeout = q->rq_timeout; 182 req->timeout = q->rq_timeout;
188 } 183
184 req->deadline = jiffies + req->timeout;
189 list_add_tail(&req->timeout_list, &q->timeout_list); 185 list_add_tail(&req->timeout_list, &q->timeout_list);
190 186
191 /* 187 /*
diff --git a/block/blk.h b/block/blk.h
index 79c85f7c9ff5..c863ec2281e0 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -13,6 +13,9 @@ extern struct kobj_type blk_queue_ktype;
13void init_request_from_bio(struct request *req, struct bio *bio); 13void init_request_from_bio(struct request *req, struct bio *bio);
14void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 14void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
15 struct bio *bio); 15 struct bio *bio);
16int blk_rq_append_bio(struct request_queue *q, struct request *rq,
17 struct bio *bio);
18void blk_dequeue_request(struct request *rq);
16void __blk_queue_free_tags(struct request_queue *q); 19void __blk_queue_free_tags(struct request_queue *q);
17 20
18void blk_unplug_work(struct work_struct *work); 21void blk_unplug_work(struct work_struct *work);
@@ -43,6 +46,43 @@ static inline void blk_clear_rq_complete(struct request *rq)
43 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); 46 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
44} 47}
45 48
49/*
50 * Internal elevator interface
51 */
52#define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash))
53
54static inline struct request *__elv_next_request(struct request_queue *q)
55{
56 struct request *rq;
57
58 while (1) {
59 while (!list_empty(&q->queue_head)) {
60 rq = list_entry_rq(q->queue_head.next);
61 if (blk_do_ordered(q, &rq))
62 return rq;
63 }
64
65 if (!q->elevator->ops->elevator_dispatch_fn(q, 0))
66 return NULL;
67 }
68}
69
70static inline void elv_activate_rq(struct request_queue *q, struct request *rq)
71{
72 struct elevator_queue *e = q->elevator;
73
74 if (e->ops->elevator_activate_req_fn)
75 e->ops->elevator_activate_req_fn(q, rq);
76}
77
78static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq)
79{
80 struct elevator_queue *e = q->elevator;
81
82 if (e->ops->elevator_deactivate_req_fn)
83 e->ops->elevator_deactivate_req_fn(q, rq);
84}
85
46#ifdef CONFIG_FAIL_IO_TIMEOUT 86#ifdef CONFIG_FAIL_IO_TIMEOUT
47int blk_should_fake_timeout(struct request_queue *); 87int blk_should_fake_timeout(struct request_queue *);
48ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 88ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
@@ -64,7 +104,6 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req,
64int attempt_back_merge(struct request_queue *q, struct request *rq); 104int attempt_back_merge(struct request_queue *q, struct request *rq);
65int attempt_front_merge(struct request_queue *q, struct request *rq); 105int attempt_front_merge(struct request_queue *q, struct request *rq);
66void blk_recalc_rq_segments(struct request *rq); 106void blk_recalc_rq_segments(struct request *rq);
67void blk_recalc_rq_sectors(struct request *rq, int nsect);
68 107
69void blk_queue_congestion_threshold(struct request_queue *q); 108void blk_queue_congestion_threshold(struct request_queue *q);
70 109
@@ -112,9 +151,17 @@ static inline int blk_cpu_to_group(int cpu)
112#endif 151#endif
113} 152}
114 153
154/*
155 * Contribute to IO statistics IFF:
156 *
157 * a) it's attached to a gendisk, and
158 * b) the queue had IO stats enabled when this request was started, and
159 * c) it's a file system request
160 */
115static inline int blk_do_io_stat(struct request *rq) 161static inline int blk_do_io_stat(struct request *rq)
116{ 162{
117 return rq->rq_disk && blk_rq_io_stat(rq); 163 return rq->rq_disk && blk_rq_io_stat(rq) && blk_fs_request(rq) &&
164 blk_discard_rq(rq);
118} 165}
119 166
120#endif 167#endif
diff --git a/block/bsg.c b/block/bsg.c
index 206060e795da..2d746e34f4c2 100644
--- a/block/bsg.c
+++ b/block/bsg.c
@@ -445,14 +445,14 @@ static int blk_complete_sgv4_hdr_rq(struct request *rq, struct sg_io_v4 *hdr,
445 } 445 }
446 446
447 if (rq->next_rq) { 447 if (rq->next_rq) {
448 hdr->dout_resid = rq->data_len; 448 hdr->dout_resid = rq->resid_len;
449 hdr->din_resid = rq->next_rq->data_len; 449 hdr->din_resid = rq->next_rq->resid_len;
450 blk_rq_unmap_user(bidi_bio); 450 blk_rq_unmap_user(bidi_bio);
451 blk_put_request(rq->next_rq); 451 blk_put_request(rq->next_rq);
452 } else if (rq_data_dir(rq) == READ) 452 } else if (rq_data_dir(rq) == READ)
453 hdr->din_resid = rq->data_len; 453 hdr->din_resid = rq->resid_len;
454 else 454 else
455 hdr->dout_resid = rq->data_len; 455 hdr->dout_resid = rq->resid_len;
456 456
457 /* 457 /*
458 * If the request generated a negative error number, return it 458 * If the request generated a negative error number, return it
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index a55a9bd75bd1..99ac4304d711 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -349,8 +349,8 @@ cfq_choose_req(struct cfq_data *cfqd, struct request *rq1, struct request *rq2)
349 else if (rq_is_meta(rq2) && !rq_is_meta(rq1)) 349 else if (rq_is_meta(rq2) && !rq_is_meta(rq1))
350 return rq2; 350 return rq2;
351 351
352 s1 = rq1->sector; 352 s1 = blk_rq_pos(rq1);
353 s2 = rq2->sector; 353 s2 = blk_rq_pos(rq2);
354 354
355 last = cfqd->last_position; 355 last = cfqd->last_position;
356 356
@@ -579,9 +579,9 @@ cfq_prio_tree_lookup(struct cfq_data *cfqd, struct rb_root *root,
579 * Sort strictly based on sector. Smallest to the left, 579 * Sort strictly based on sector. Smallest to the left,
580 * largest to the right. 580 * largest to the right.
581 */ 581 */
582 if (sector > cfqq->next_rq->sector) 582 if (sector > blk_rq_pos(cfqq->next_rq))
583 n = &(*p)->rb_right; 583 n = &(*p)->rb_right;
584 else if (sector < cfqq->next_rq->sector) 584 else if (sector < blk_rq_pos(cfqq->next_rq))
585 n = &(*p)->rb_left; 585 n = &(*p)->rb_left;
586 else 586 else
587 break; 587 break;
@@ -611,8 +611,8 @@ static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)
611 return; 611 return;
612 612
613 cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio]; 613 cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio];
614 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root, cfqq->next_rq->sector, 614 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root,
615 &parent, &p); 615 blk_rq_pos(cfqq->next_rq), &parent, &p);
616 if (!__cfqq) { 616 if (!__cfqq) {
617 rb_link_node(&cfqq->p_node, parent, p); 617 rb_link_node(&cfqq->p_node, parent, p);
618 rb_insert_color(&cfqq->p_node, cfqq->p_root); 618 rb_insert_color(&cfqq->p_node, cfqq->p_root);
@@ -760,7 +760,7 @@ static void cfq_activate_request(struct request_queue *q, struct request *rq)
760 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "activate rq, drv=%d", 760 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "activate rq, drv=%d",
761 cfqd->rq_in_driver); 761 cfqd->rq_in_driver);
762 762
763 cfqd->last_position = rq->hard_sector + rq->hard_nr_sectors; 763 cfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
764} 764}
765 765
766static void cfq_deactivate_request(struct request_queue *q, struct request *rq) 766static void cfq_deactivate_request(struct request_queue *q, struct request *rq)
@@ -949,10 +949,10 @@ static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd,
949static inline sector_t cfq_dist_from_last(struct cfq_data *cfqd, 949static inline sector_t cfq_dist_from_last(struct cfq_data *cfqd,
950 struct request *rq) 950 struct request *rq)
951{ 951{
952 if (rq->sector >= cfqd->last_position) 952 if (blk_rq_pos(rq) >= cfqd->last_position)
953 return rq->sector - cfqd->last_position; 953 return blk_rq_pos(rq) - cfqd->last_position;
954 else 954 else
955 return cfqd->last_position - rq->sector; 955 return cfqd->last_position - blk_rq_pos(rq);
956} 956}
957 957
958#define CIC_SEEK_THR 8 * 1024 958#define CIC_SEEK_THR 8 * 1024
@@ -996,7 +996,7 @@ static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
996 if (cfq_rq_close(cfqd, __cfqq->next_rq)) 996 if (cfq_rq_close(cfqd, __cfqq->next_rq))
997 return __cfqq; 997 return __cfqq;
998 998
999 if (__cfqq->next_rq->sector < sector) 999 if (blk_rq_pos(__cfqq->next_rq) < sector)
1000 node = rb_next(&__cfqq->p_node); 1000 node = rb_next(&__cfqq->p_node);
1001 else 1001 else
1002 node = rb_prev(&__cfqq->p_node); 1002 node = rb_prev(&__cfqq->p_node);
@@ -1918,10 +1918,10 @@ cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_io_context *cic,
1918 1918
1919 if (!cic->last_request_pos) 1919 if (!cic->last_request_pos)
1920 sdist = 0; 1920 sdist = 0;
1921 else if (cic->last_request_pos < rq->sector) 1921 else if (cic->last_request_pos < blk_rq_pos(rq))
1922 sdist = rq->sector - cic->last_request_pos; 1922 sdist = blk_rq_pos(rq) - cic->last_request_pos;
1923 else 1923 else
1924 sdist = cic->last_request_pos - rq->sector; 1924 sdist = cic->last_request_pos - blk_rq_pos(rq);
1925 1925
1926 /* 1926 /*
1927 * Don't allow the seek distance to get too large from the 1927 * Don't allow the seek distance to get too large from the
@@ -2071,7 +2071,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
2071 cfq_update_io_seektime(cfqd, cic, rq); 2071 cfq_update_io_seektime(cfqd, cic, rq);
2072 cfq_update_idle_window(cfqd, cfqq, cic); 2072 cfq_update_idle_window(cfqd, cfqq, cic);
2073 2073
2074 cic->last_request_pos = rq->sector + rq->nr_sectors; 2074 cic->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
2075 2075
2076 if (cfqq == cfqd->active_queue) { 2076 if (cfqq == cfqd->active_queue) {
2077 /* 2077 /*
@@ -2088,7 +2088,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
2088 if (blk_rq_bytes(rq) > PAGE_CACHE_SIZE || 2088 if (blk_rq_bytes(rq) > PAGE_CACHE_SIZE ||
2089 cfqd->busy_queues > 1) { 2089 cfqd->busy_queues > 1) {
2090 del_timer(&cfqd->idle_slice_timer); 2090 del_timer(&cfqd->idle_slice_timer);
2091 blk_start_queueing(cfqd->queue); 2091 __blk_run_queue(cfqd->queue);
2092 } 2092 }
2093 cfq_mark_cfqq_must_dispatch(cfqq); 2093 cfq_mark_cfqq_must_dispatch(cfqq);
2094 } 2094 }
@@ -2100,7 +2100,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
2100 * this new queue is RT and the current one is BE 2100 * this new queue is RT and the current one is BE
2101 */ 2101 */
2102 cfq_preempt_queue(cfqd, cfqq); 2102 cfq_preempt_queue(cfqd, cfqq);
2103 blk_start_queueing(cfqd->queue); 2103 __blk_run_queue(cfqd->queue);
2104 } 2104 }
2105} 2105}
2106 2106
@@ -2345,7 +2345,7 @@ static void cfq_kick_queue(struct work_struct *work)
2345 struct request_queue *q = cfqd->queue; 2345 struct request_queue *q = cfqd->queue;
2346 2346
2347 spin_lock_irq(q->queue_lock); 2347 spin_lock_irq(q->queue_lock);
2348 blk_start_queueing(q); 2348 __blk_run_queue(cfqd->queue);
2349 spin_unlock_irq(q->queue_lock); 2349 spin_unlock_irq(q->queue_lock);
2350} 2350}
2351 2351
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c
index c4d991d4adef..b547cbca7b23 100644
--- a/block/deadline-iosched.c
+++ b/block/deadline-iosched.c
@@ -138,7 +138,7 @@ deadline_merge(struct request_queue *q, struct request **req, struct bio *bio)
138 138
139 __rq = elv_rb_find(&dd->sort_list[bio_data_dir(bio)], sector); 139 __rq = elv_rb_find(&dd->sort_list[bio_data_dir(bio)], sector);
140 if (__rq) { 140 if (__rq) {
141 BUG_ON(sector != __rq->sector); 141 BUG_ON(sector != blk_rq_pos(__rq));
142 142
143 if (elv_rq_merge_ok(__rq, bio)) { 143 if (elv_rq_merge_ok(__rq, bio)) {
144 ret = ELEVATOR_FRONT_MERGE; 144 ret = ELEVATOR_FRONT_MERGE;
diff --git a/block/elevator.c b/block/elevator.c
index 7073a9072577..ebee948293eb 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -52,8 +52,7 @@ static const int elv_hash_shift = 6;
52#define ELV_HASH_FN(sec) \ 52#define ELV_HASH_FN(sec) \
53 (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift)) 53 (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift))
54#define ELV_HASH_ENTRIES (1 << elv_hash_shift) 54#define ELV_HASH_ENTRIES (1 << elv_hash_shift)
55#define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors) 55#define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
56#define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash))
57 56
58DEFINE_TRACE(block_rq_insert); 57DEFINE_TRACE(block_rq_insert);
59DEFINE_TRACE(block_rq_issue); 58DEFINE_TRACE(block_rq_issue);
@@ -120,9 +119,9 @@ static inline int elv_try_merge(struct request *__rq, struct bio *bio)
120 * we can merge and sequence is ok, check if it's possible 119 * we can merge and sequence is ok, check if it's possible
121 */ 120 */
122 if (elv_rq_merge_ok(__rq, bio)) { 121 if (elv_rq_merge_ok(__rq, bio)) {
123 if (__rq->sector + __rq->nr_sectors == bio->bi_sector) 122 if (blk_rq_pos(__rq) + blk_rq_sectors(__rq) == bio->bi_sector)
124 ret = ELEVATOR_BACK_MERGE; 123 ret = ELEVATOR_BACK_MERGE;
125 else if (__rq->sector - bio_sectors(bio) == bio->bi_sector) 124 else if (blk_rq_pos(__rq) - bio_sectors(bio) == bio->bi_sector)
126 ret = ELEVATOR_FRONT_MERGE; 125 ret = ELEVATOR_FRONT_MERGE;
127 } 126 }
128 127
@@ -310,22 +309,6 @@ void elevator_exit(struct elevator_queue *e)
310} 309}
311EXPORT_SYMBOL(elevator_exit); 310EXPORT_SYMBOL(elevator_exit);
312 311
313static void elv_activate_rq(struct request_queue *q, struct request *rq)
314{
315 struct elevator_queue *e = q->elevator;
316
317 if (e->ops->elevator_activate_req_fn)
318 e->ops->elevator_activate_req_fn(q, rq);
319}
320
321static void elv_deactivate_rq(struct request_queue *q, struct request *rq)
322{
323 struct elevator_queue *e = q->elevator;
324
325 if (e->ops->elevator_deactivate_req_fn)
326 e->ops->elevator_deactivate_req_fn(q, rq);
327}
328
329static inline void __elv_rqhash_del(struct request *rq) 312static inline void __elv_rqhash_del(struct request *rq)
330{ 313{
331 hlist_del_init(&rq->hash); 314 hlist_del_init(&rq->hash);
@@ -387,9 +370,9 @@ struct request *elv_rb_add(struct rb_root *root, struct request *rq)
387 parent = *p; 370 parent = *p;
388 __rq = rb_entry(parent, struct request, rb_node); 371 __rq = rb_entry(parent, struct request, rb_node);
389 372
390 if (rq->sector < __rq->sector) 373 if (blk_rq_pos(rq) < blk_rq_pos(__rq))
391 p = &(*p)->rb_left; 374 p = &(*p)->rb_left;
392 else if (rq->sector > __rq->sector) 375 else if (blk_rq_pos(rq) > blk_rq_pos(__rq))
393 p = &(*p)->rb_right; 376 p = &(*p)->rb_right;
394 else 377 else
395 return __rq; 378 return __rq;
@@ -417,9 +400,9 @@ struct request *elv_rb_find(struct rb_root *root, sector_t sector)
417 while (n) { 400 while (n) {
418 rq = rb_entry(n, struct request, rb_node); 401 rq = rb_entry(n, struct request, rb_node);
419 402
420 if (sector < rq->sector) 403 if (sector < blk_rq_pos(rq))
421 n = n->rb_left; 404 n = n->rb_left;
422 else if (sector > rq->sector) 405 else if (sector > blk_rq_pos(rq))
423 n = n->rb_right; 406 n = n->rb_right;
424 else 407 else
425 return rq; 408 return rq;
@@ -458,14 +441,14 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq)
458 break; 441 break;
459 if (pos->cmd_flags & stop_flags) 442 if (pos->cmd_flags & stop_flags)
460 break; 443 break;
461 if (rq->sector >= boundary) { 444 if (blk_rq_pos(rq) >= boundary) {
462 if (pos->sector < boundary) 445 if (blk_rq_pos(pos) < boundary)
463 continue; 446 continue;
464 } else { 447 } else {
465 if (pos->sector >= boundary) 448 if (blk_rq_pos(pos) >= boundary)
466 break; 449 break;
467 } 450 }
468 if (rq->sector >= pos->sector) 451 if (blk_rq_pos(rq) >= blk_rq_pos(pos))
469 break; 452 break;
470 } 453 }
471 454
@@ -563,7 +546,7 @@ void elv_requeue_request(struct request_queue *q, struct request *rq)
563 * in_flight count again 546 * in_flight count again
564 */ 547 */
565 if (blk_account_rq(rq)) { 548 if (blk_account_rq(rq)) {
566 q->in_flight--; 549 q->in_flight[rq_is_sync(rq)]--;
567 if (blk_sorted_rq(rq)) 550 if (blk_sorted_rq(rq))
568 elv_deactivate_rq(q, rq); 551 elv_deactivate_rq(q, rq);
569 } 552 }
@@ -599,7 +582,7 @@ void elv_quiesce_start(struct request_queue *q)
599 */ 582 */
600 elv_drain_elevator(q); 583 elv_drain_elevator(q);
601 while (q->rq.elvpriv) { 584 while (q->rq.elvpriv) {
602 blk_start_queueing(q); 585 __blk_run_queue(q);
603 spin_unlock_irq(q->queue_lock); 586 spin_unlock_irq(q->queue_lock);
604 msleep(10); 587 msleep(10);
605 spin_lock_irq(q->queue_lock); 588 spin_lock_irq(q->queue_lock);
@@ -643,8 +626,7 @@ void elv_insert(struct request_queue *q, struct request *rq, int where)
643 * with anything. There's no point in delaying queue 626 * with anything. There's no point in delaying queue
644 * processing. 627 * processing.
645 */ 628 */
646 blk_remove_plug(q); 629 __blk_run_queue(q);
647 blk_start_queueing(q);
648 break; 630 break;
649 631
650 case ELEVATOR_INSERT_SORT: 632 case ELEVATOR_INSERT_SORT:
@@ -703,7 +685,7 @@ void elv_insert(struct request_queue *q, struct request *rq, int where)
703 685
704 if (unplug_it && blk_queue_plugged(q)) { 686 if (unplug_it && blk_queue_plugged(q)) {
705 int nrq = q->rq.count[BLK_RW_SYNC] + q->rq.count[BLK_RW_ASYNC] 687 int nrq = q->rq.count[BLK_RW_SYNC] + q->rq.count[BLK_RW_ASYNC]
706 - q->in_flight; 688 - queue_in_flight(q);
707 689
708 if (nrq >= q->unplug_thresh) 690 if (nrq >= q->unplug_thresh)
709 __generic_unplug_device(q); 691 __generic_unplug_device(q);
@@ -759,117 +741,6 @@ void elv_add_request(struct request_queue *q, struct request *rq, int where,
759} 741}
760EXPORT_SYMBOL(elv_add_request); 742EXPORT_SYMBOL(elv_add_request);
761 743
762static inline struct request *__elv_next_request(struct request_queue *q)
763{
764 struct request *rq;
765
766 while (1) {
767 while (!list_empty(&q->queue_head)) {
768 rq = list_entry_rq(q->queue_head.next);
769 if (blk_do_ordered(q, &rq))
770 return rq;
771 }
772
773 if (!q->elevator->ops->elevator_dispatch_fn(q, 0))
774 return NULL;
775 }
776}
777
778struct request *elv_next_request(struct request_queue *q)
779{
780 struct request *rq;
781 int ret;
782
783 while ((rq = __elv_next_request(q)) != NULL) {
784 if (!(rq->cmd_flags & REQ_STARTED)) {
785 /*
786 * This is the first time the device driver
787 * sees this request (possibly after
788 * requeueing). Notify IO scheduler.
789 */
790 if (blk_sorted_rq(rq))
791 elv_activate_rq(q, rq);
792
793 /*
794 * just mark as started even if we don't start
795 * it, a request that has been delayed should
796 * not be passed by new incoming requests
797 */
798 rq->cmd_flags |= REQ_STARTED;
799 trace_block_rq_issue(q, rq);
800 }
801
802 if (!q->boundary_rq || q->boundary_rq == rq) {
803 q->end_sector = rq_end_sector(rq);
804 q->boundary_rq = NULL;
805 }
806
807 if (rq->cmd_flags & REQ_DONTPREP)
808 break;
809
810 if (q->dma_drain_size && rq->data_len) {
811 /*
812 * make sure space for the drain appears we
813 * know we can do this because max_hw_segments
814 * has been adjusted to be one fewer than the
815 * device can handle
816 */
817 rq->nr_phys_segments++;
818 }
819
820 if (!q->prep_rq_fn)
821 break;
822
823 ret = q->prep_rq_fn(q, rq);
824 if (ret == BLKPREP_OK) {
825 break;
826 } else if (ret == BLKPREP_DEFER) {
827 /*
828 * the request may have been (partially) prepped.
829 * we need to keep this request in the front to
830 * avoid resource deadlock. REQ_STARTED will
831 * prevent other fs requests from passing this one.
832 */
833 if (q->dma_drain_size && rq->data_len &&
834 !(rq->cmd_flags & REQ_DONTPREP)) {
835 /*
836 * remove the space for the drain we added
837 * so that we don't add it again
838 */
839 --rq->nr_phys_segments;
840 }
841
842 rq = NULL;
843 break;
844 } else if (ret == BLKPREP_KILL) {
845 rq->cmd_flags |= REQ_QUIET;
846 __blk_end_request(rq, -EIO, blk_rq_bytes(rq));
847 } else {
848 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
849 break;
850 }
851 }
852
853 return rq;
854}
855EXPORT_SYMBOL(elv_next_request);
856
857void elv_dequeue_request(struct request_queue *q, struct request *rq)
858{
859 BUG_ON(list_empty(&rq->queuelist));
860 BUG_ON(ELV_ON_HASH(rq));
861
862 list_del_init(&rq->queuelist);
863
864 /*
865 * the time frame between a request being removed from the lists
866 * and to it is freed is accounted as io that is in progress at
867 * the driver side.
868 */
869 if (blk_account_rq(rq))
870 q->in_flight++;
871}
872
873int elv_queue_empty(struct request_queue *q) 744int elv_queue_empty(struct request_queue *q)
874{ 745{
875 struct elevator_queue *e = q->elevator; 746 struct elevator_queue *e = q->elevator;
@@ -939,7 +810,7 @@ void elv_abort_queue(struct request_queue *q)
939 rq = list_entry_rq(q->queue_head.next); 810 rq = list_entry_rq(q->queue_head.next);
940 rq->cmd_flags |= REQ_QUIET; 811 rq->cmd_flags |= REQ_QUIET;
941 trace_block_rq_abort(q, rq); 812 trace_block_rq_abort(q, rq);
942 __blk_end_request(rq, -EIO, blk_rq_bytes(rq)); 813 __blk_end_request_all(rq, -EIO);
943 } 814 }
944} 815}
945EXPORT_SYMBOL(elv_abort_queue); 816EXPORT_SYMBOL(elv_abort_queue);
@@ -952,7 +823,7 @@ void elv_completed_request(struct request_queue *q, struct request *rq)
952 * request is released from the driver, io must be done 823 * request is released from the driver, io must be done
953 */ 824 */
954 if (blk_account_rq(rq)) { 825 if (blk_account_rq(rq)) {
955 q->in_flight--; 826 q->in_flight[rq_is_sync(rq)]--;
956 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn) 827 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
957 e->ops->elevator_completed_req_fn(q, rq); 828 e->ops->elevator_completed_req_fn(q, rq);
958 } 829 }
@@ -967,11 +838,11 @@ void elv_completed_request(struct request_queue *q, struct request *rq)
967 if (!list_empty(&q->queue_head)) 838 if (!list_empty(&q->queue_head))
968 next = list_entry_rq(q->queue_head.next); 839 next = list_entry_rq(q->queue_head.next);
969 840
970 if (!q->in_flight && 841 if (!queue_in_flight(q) &&
971 blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN && 842 blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN &&
972 (!next || blk_ordered_req_seq(next) > QUEUE_ORDSEQ_DRAIN)) { 843 (!next || blk_ordered_req_seq(next) > QUEUE_ORDSEQ_DRAIN)) {
973 blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0); 844 blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0);
974 blk_start_queueing(q); 845 __blk_run_queue(q);
975 } 846 }
976 } 847 }
977} 848}
diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c
index 82a0ca2f6729..a9670dd4b5de 100644
--- a/block/scsi_ioctl.c
+++ b/block/scsi_ioctl.c
@@ -230,7 +230,7 @@ static int blk_complete_sghdr_rq(struct request *rq, struct sg_io_hdr *hdr,
230 hdr->info = 0; 230 hdr->info = 0;
231 if (hdr->masked_status || hdr->host_status || hdr->driver_status) 231 if (hdr->masked_status || hdr->host_status || hdr->driver_status)
232 hdr->info |= SG_INFO_CHECK; 232 hdr->info |= SG_INFO_CHECK;
233 hdr->resid = rq->data_len; 233 hdr->resid = rq->resid_len;
234 hdr->sb_len_wr = 0; 234 hdr->sb_len_wr = 0;
235 235
236 if (rq->sense_len && hdr->sbp) { 236 if (rq->sense_len && hdr->sbp) {
@@ -500,9 +500,6 @@ static int __blk_send_generic(struct request_queue *q, struct gendisk *bd_disk,
500 500
501 rq = blk_get_request(q, WRITE, __GFP_WAIT); 501 rq = blk_get_request(q, WRITE, __GFP_WAIT);
502 rq->cmd_type = REQ_TYPE_BLOCK_PC; 502 rq->cmd_type = REQ_TYPE_BLOCK_PC;
503 rq->data = NULL;
504 rq->data_len = 0;
505 rq->extra_len = 0;
506 rq->timeout = BLK_DEFAULT_SG_TIMEOUT; 503 rq->timeout = BLK_DEFAULT_SG_TIMEOUT;
507 rq->cmd[0] = cmd; 504 rq->cmd[0] = cmd;
508 rq->cmd[4] = data; 505 rq->cmd[4] = data;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-22 20:16:09 -0500