aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/block
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/block')
-rw-r--r--drivers/block/as-iosched.c325
-rw-r--r--drivers/block/cfq-iosched.c364
-rw-r--r--drivers/block/deadline-iosched.c123
-rw-r--r--drivers/block/elevator.c266
-rw-r--r--drivers/block/ll_rw_blk.c23
-rw-r--r--drivers/block/noop-iosched.c48
6 files changed, 376 insertions, 773 deletions
diff --git a/drivers/block/as-iosched.c b/drivers/block/as-iosched.c
index 1f08e14697e9..4081c36c8c19 100644
--- a/drivers/block/as-iosched.c
+++ b/drivers/block/as-iosched.c
@@ -98,7 +98,6 @@ struct as_data {
98 98
99 struct as_rq *next_arq[2]; /* next in sort order */ 99 struct as_rq *next_arq[2]; /* next in sort order */
100 sector_t last_sector[2]; /* last REQ_SYNC & REQ_ASYNC sectors */ 100 sector_t last_sector[2]; /* last REQ_SYNC & REQ_ASYNC sectors */
101 struct list_head *dispatch; /* driver dispatch queue */
102 struct list_head *hash; /* request hash */ 101 struct list_head *hash; /* request hash */
103 102
104 unsigned long exit_prob; /* probability a task will exit while 103 unsigned long exit_prob; /* probability a task will exit while
@@ -239,6 +238,25 @@ static struct io_context *as_get_io_context(void)
239 return ioc; 238 return ioc;
240} 239}
241 240
241static void as_put_io_context(struct as_rq *arq)
242{
243 struct as_io_context *aic;
244
245 if (unlikely(!arq->io_context))
246 return;
247
248 aic = arq->io_context->aic;
249
250 if (arq->is_sync == REQ_SYNC && aic) {
251 spin_lock(&aic->lock);
252 set_bit(AS_TASK_IORUNNING, &aic->state);
253 aic->last_end_request = jiffies;
254 spin_unlock(&aic->lock);
255 }
256
257 put_io_context(arq->io_context);
258}
259
242/* 260/*
243 * the back merge hash support functions 261 * the back merge hash support functions
244 */ 262 */
@@ -261,14 +279,6 @@ static inline void as_del_arq_hash(struct as_rq *arq)
261 __as_del_arq_hash(arq); 279 __as_del_arq_hash(arq);
262} 280}
263 281
264static void as_remove_merge_hints(request_queue_t *q, struct as_rq *arq)
265{
266 as_del_arq_hash(arq);
267
268 if (q->last_merge == arq->request)
269 q->last_merge = NULL;
270}
271
272static void as_add_arq_hash(struct as_data *ad, struct as_rq *arq) 282static void as_add_arq_hash(struct as_data *ad, struct as_rq *arq)
273{ 283{
274 struct request *rq = arq->request; 284 struct request *rq = arq->request;
@@ -312,7 +322,7 @@ static struct request *as_find_arq_hash(struct as_data *ad, sector_t offset)
312 BUG_ON(!arq->on_hash); 322 BUG_ON(!arq->on_hash);
313 323
314 if (!rq_mergeable(__rq)) { 324 if (!rq_mergeable(__rq)) {
315 as_remove_merge_hints(ad->q, arq); 325 as_del_arq_hash(arq);
316 continue; 326 continue;
317 } 327 }
318 328
@@ -950,23 +960,12 @@ static void as_completed_request(request_queue_t *q, struct request *rq)
950 960
951 WARN_ON(!list_empty(&rq->queuelist)); 961 WARN_ON(!list_empty(&rq->queuelist));
952 962
953 if (arq->state == AS_RQ_PRESCHED) {
954 WARN_ON(arq->io_context);
955 goto out;
956 }
957
958 if (arq->state == AS_RQ_MERGED)
959 goto out_ioc;
960
961 if (arq->state != AS_RQ_REMOVED) { 963 if (arq->state != AS_RQ_REMOVED) {
962 printk("arq->state %d\n", arq->state); 964 printk("arq->state %d\n", arq->state);
963 WARN_ON(1); 965 WARN_ON(1);
964 goto out; 966 goto out;
965 } 967 }
966 968
967 if (!blk_fs_request(rq))
968 goto out;
969
970 if (ad->changed_batch && ad->nr_dispatched == 1) { 969 if (ad->changed_batch && ad->nr_dispatched == 1) {
971 kblockd_schedule_work(&ad->antic_work); 970 kblockd_schedule_work(&ad->antic_work);
972 ad->changed_batch = 0; 971 ad->changed_batch = 0;
@@ -1001,21 +1000,7 @@ static void as_completed_request(request_queue_t *q, struct request *rq)
1001 } 1000 }
1002 } 1001 }
1003 1002
1004out_ioc: 1003 as_put_io_context(arq);
1005 if (!arq->io_context)
1006 goto out;
1007
1008 if (arq->is_sync == REQ_SYNC) {
1009 struct as_io_context *aic = arq->io_context->aic;
1010 if (aic) {
1011 spin_lock(&aic->lock);
1012 set_bit(AS_TASK_IORUNNING, &aic->state);
1013 aic->last_end_request = jiffies;
1014 spin_unlock(&aic->lock);
1015 }
1016 }
1017
1018 put_io_context(arq->io_context);
1019out: 1004out:
1020 arq->state = AS_RQ_POSTSCHED; 1005 arq->state = AS_RQ_POSTSCHED;
1021} 1006}
@@ -1047,73 +1032,11 @@ static void as_remove_queued_request(request_queue_t *q, struct request *rq)
1047 ad->next_arq[data_dir] = as_find_next_arq(ad, arq); 1032 ad->next_arq[data_dir] = as_find_next_arq(ad, arq);
1048 1033
1049 list_del_init(&arq->fifo); 1034 list_del_init(&arq->fifo);
1050 as_remove_merge_hints(q, arq); 1035 as_del_arq_hash(arq);
1051 as_del_arq_rb(ad, arq); 1036 as_del_arq_rb(ad, arq);
1052} 1037}
1053 1038
1054/* 1039/*
1055 * as_remove_dispatched_request is called to remove a request which has gone
1056 * to the dispatch list.
1057 */
1058static void as_remove_dispatched_request(request_queue_t *q, struct request *rq)
1059{
1060 struct as_rq *arq = RQ_DATA(rq);
1061 struct as_io_context *aic;
1062
1063 if (!arq) {
1064 WARN_ON(1);
1065 return;
1066 }
1067
1068 WARN_ON(arq->state != AS_RQ_DISPATCHED);
1069 WARN_ON(ON_RB(&arq->rb_node));
1070 if (arq->io_context && arq->io_context->aic) {
1071 aic = arq->io_context->aic;
1072 if (aic) {
1073 WARN_ON(!atomic_read(&aic->nr_dispatched));
1074 atomic_dec(&aic->nr_dispatched);
1075 }
1076 }
1077}
1078
1079/*
1080 * as_remove_request is called when a driver has finished with a request.
1081 * This should be only called for dispatched requests, but for some reason
1082 * a POWER4 box running hwscan it does not.
1083 */
1084static void as_remove_request(request_queue_t *q, struct request *rq)
1085{
1086 struct as_rq *arq = RQ_DATA(rq);
1087
1088 if (unlikely(arq->state == AS_RQ_NEW))
1089 goto out;
1090
1091 if (ON_RB(&arq->rb_node)) {
1092 if (arq->state != AS_RQ_QUEUED) {
1093 printk("arq->state %d\n", arq->state);
1094 WARN_ON(1);
1095 goto out;
1096 }
1097 /*
1098 * We'll lose the aliased request(s) here. I don't think this
1099 * will ever happen, but if it does, hopefully someone will
1100 * report it.
1101 */
1102 WARN_ON(!list_empty(&rq->queuelist));
1103 as_remove_queued_request(q, rq);
1104 } else {
1105 if (arq->state != AS_RQ_DISPATCHED) {
1106 printk("arq->state %d\n", arq->state);
1107 WARN_ON(1);
1108 goto out;
1109 }
1110 as_remove_dispatched_request(q, rq);
1111 }
1112out:
1113 arq->state = AS_RQ_REMOVED;
1114}
1115
1116/*
1117 * as_fifo_expired returns 0 if there are no expired reads on the fifo, 1040 * as_fifo_expired returns 0 if there are no expired reads on the fifo,
1118 * 1 otherwise. It is ratelimited so that we only perform the check once per 1041 * 1 otherwise. It is ratelimited so that we only perform the check once per
1119 * `fifo_expire' interval. Otherwise a large number of expired requests 1042 * `fifo_expire' interval. Otherwise a large number of expired requests
@@ -1165,7 +1088,6 @@ static inline int as_batch_expired(struct as_data *ad)
1165static void as_move_to_dispatch(struct as_data *ad, struct as_rq *arq) 1088static void as_move_to_dispatch(struct as_data *ad, struct as_rq *arq)
1166{ 1089{
1167 struct request *rq = arq->request; 1090 struct request *rq = arq->request;
1168 struct list_head *insert;
1169 const int data_dir = arq->is_sync; 1091 const int data_dir = arq->is_sync;
1170 1092
1171 BUG_ON(!ON_RB(&arq->rb_node)); 1093 BUG_ON(!ON_RB(&arq->rb_node));
@@ -1198,13 +1120,13 @@ static void as_move_to_dispatch(struct as_data *ad, struct as_rq *arq)
1198 /* 1120 /*
1199 * take it off the sort and fifo list, add to dispatch queue 1121 * take it off the sort and fifo list, add to dispatch queue
1200 */ 1122 */
1201 insert = ad->dispatch->prev;
1202
1203 while (!list_empty(&rq->queuelist)) { 1123 while (!list_empty(&rq->queuelist)) {
1204 struct request *__rq = list_entry_rq(rq->queuelist.next); 1124 struct request *__rq = list_entry_rq(rq->queuelist.next);
1205 struct as_rq *__arq = RQ_DATA(__rq); 1125 struct as_rq *__arq = RQ_DATA(__rq);
1206 1126
1207 list_move_tail(&__rq->queuelist, ad->dispatch); 1127 list_del(&__rq->queuelist);
1128
1129 elv_dispatch_add_tail(ad->q, __rq);
1208 1130
1209 if (__arq->io_context && __arq->io_context->aic) 1131 if (__arq->io_context && __arq->io_context->aic)
1210 atomic_inc(&__arq->io_context->aic->nr_dispatched); 1132 atomic_inc(&__arq->io_context->aic->nr_dispatched);
@@ -1218,7 +1140,8 @@ static void as_move_to_dispatch(struct as_data *ad, struct as_rq *arq)
1218 as_remove_queued_request(ad->q, rq); 1140 as_remove_queued_request(ad->q, rq);
1219 WARN_ON(arq->state != AS_RQ_QUEUED); 1141 WARN_ON(arq->state != AS_RQ_QUEUED);
1220 1142
1221 list_add(&rq->queuelist, insert); 1143 elv_dispatch_sort(ad->q, rq);
1144
1222 arq->state = AS_RQ_DISPATCHED; 1145 arq->state = AS_RQ_DISPATCHED;
1223 if (arq->io_context && arq->io_context->aic) 1146 if (arq->io_context && arq->io_context->aic)
1224 atomic_inc(&arq->io_context->aic->nr_dispatched); 1147 atomic_inc(&arq->io_context->aic->nr_dispatched);
@@ -1230,12 +1153,42 @@ static void as_move_to_dispatch(struct as_data *ad, struct as_rq *arq)
1230 * read/write expire, batch expire, etc, and moves it to the dispatch 1153 * read/write expire, batch expire, etc, and moves it to the dispatch
1231 * queue. Returns 1 if a request was found, 0 otherwise. 1154 * queue. Returns 1 if a request was found, 0 otherwise.
1232 */ 1155 */
1233static int as_dispatch_request(struct as_data *ad) 1156static int as_dispatch_request(request_queue_t *q, int force)
1234{ 1157{
1158 struct as_data *ad = q->elevator->elevator_data;
1235 struct as_rq *arq; 1159 struct as_rq *arq;
1236 const int reads = !list_empty(&ad->fifo_list[REQ_SYNC]); 1160 const int reads = !list_empty(&ad->fifo_list[REQ_SYNC]);
1237 const int writes = !list_empty(&ad->fifo_list[REQ_ASYNC]); 1161 const int writes = !list_empty(&ad->fifo_list[REQ_ASYNC]);
1238 1162
1163 if (unlikely(force)) {
1164 /*
1165 * Forced dispatch, accounting is useless. Reset
1166 * accounting states and dump fifo_lists. Note that
1167 * batch_data_dir is reset to REQ_SYNC to avoid
1168 * screwing write batch accounting as write batch
1169 * accounting occurs on W->R transition.
1170 */
1171 int dispatched = 0;
1172
1173 ad->batch_data_dir = REQ_SYNC;
1174 ad->changed_batch = 0;
1175 ad->new_batch = 0;
1176
1177 while (ad->next_arq[REQ_SYNC]) {
1178 as_move_to_dispatch(ad, ad->next_arq[REQ_SYNC]);
1179 dispatched++;
1180 }
1181 ad->last_check_fifo[REQ_SYNC] = jiffies;
1182
1183 while (ad->next_arq[REQ_ASYNC]) {
1184 as_move_to_dispatch(ad, ad->next_arq[REQ_ASYNC]);
1185 dispatched++;
1186 }
1187 ad->last_check_fifo[REQ_ASYNC] = jiffies;
1188
1189 return dispatched;
1190 }
1191
1239 /* Signal that the write batch was uncontended, so we can't time it */ 1192 /* Signal that the write batch was uncontended, so we can't time it */
1240 if (ad->batch_data_dir == REQ_ASYNC && !reads) { 1193 if (ad->batch_data_dir == REQ_ASYNC && !reads) {
1241 if (ad->current_write_count == 0 || !writes) 1194 if (ad->current_write_count == 0 || !writes)
@@ -1359,20 +1312,6 @@ fifo_expired:
1359 return 1; 1312 return 1;
1360} 1313}
1361 1314
1362static struct request *as_next_request(request_queue_t *q)
1363{
1364 struct as_data *ad = q->elevator->elevator_data;
1365 struct request *rq = NULL;
1366
1367 /*
1368 * if there are still requests on the dispatch queue, grab the first
1369 */
1370 if (!list_empty(ad->dispatch) || as_dispatch_request(ad))
1371 rq = list_entry_rq(ad->dispatch->next);
1372
1373 return rq;
1374}
1375
1376/* 1315/*
1377 * Add arq to a list behind alias 1316 * Add arq to a list behind alias
1378 */ 1317 */
@@ -1404,17 +1343,25 @@ as_add_aliased_request(struct as_data *ad, struct as_rq *arq, struct as_rq *alia
1404 /* 1343 /*
1405 * Don't want to have to handle merges. 1344 * Don't want to have to handle merges.
1406 */ 1345 */
1407 as_remove_merge_hints(ad->q, arq); 1346 as_del_arq_hash(arq);
1408} 1347}
1409 1348
1410/* 1349/*
1411 * add arq to rbtree and fifo 1350 * add arq to rbtree and fifo
1412 */ 1351 */
1413static void as_add_request(struct as_data *ad, struct as_rq *arq) 1352static void as_add_request(request_queue_t *q, struct request *rq)
1414{ 1353{
1354 struct as_data *ad = q->elevator->elevator_data;
1355 struct as_rq *arq = RQ_DATA(rq);
1415 struct as_rq *alias; 1356 struct as_rq *alias;
1416 int data_dir; 1357 int data_dir;
1417 1358
1359 if (arq->state != AS_RQ_PRESCHED) {
1360 printk("arq->state: %d\n", arq->state);
1361 WARN_ON(1);
1362 }
1363 arq->state = AS_RQ_NEW;
1364
1418 if (rq_data_dir(arq->request) == READ 1365 if (rq_data_dir(arq->request) == READ
1419 || current->flags&PF_SYNCWRITE) 1366 || current->flags&PF_SYNCWRITE)
1420 arq->is_sync = 1; 1367 arq->is_sync = 1;
@@ -1437,12 +1384,8 @@ static void as_add_request(struct as_data *ad, struct as_rq *arq)
1437 arq->expires = jiffies + ad->fifo_expire[data_dir]; 1384 arq->expires = jiffies + ad->fifo_expire[data_dir];
1438 list_add_tail(&arq->fifo, &ad->fifo_list[data_dir]); 1385 list_add_tail(&arq->fifo, &ad->fifo_list[data_dir]);
1439 1386
1440 if (rq_mergeable(arq->request)) { 1387 if (rq_mergeable(arq->request))
1441 as_add_arq_hash(ad, arq); 1388 as_add_arq_hash(ad, arq);
1442
1443 if (!ad->q->last_merge)
1444 ad->q->last_merge = arq->request;
1445 }
1446 as_update_arq(ad, arq); /* keep state machine up to date */ 1389 as_update_arq(ad, arq); /* keep state machine up to date */
1447 1390
1448 } else { 1391 } else {
@@ -1463,96 +1406,24 @@ static void as_add_request(struct as_data *ad, struct as_rq *arq)
1463 arq->state = AS_RQ_QUEUED; 1406 arq->state = AS_RQ_QUEUED;
1464} 1407}
1465 1408
1466static void as_deactivate_request(request_queue_t *q, struct request *rq) 1409static void as_activate_request(request_queue_t *q, struct request *rq)
1467{ 1410{
1468 struct as_data *ad = q->elevator->elevator_data;
1469 struct as_rq *arq = RQ_DATA(rq); 1411 struct as_rq *arq = RQ_DATA(rq);
1470 1412
1471 if (arq) { 1413 WARN_ON(arq->state != AS_RQ_DISPATCHED);
1472 if (arq->state == AS_RQ_REMOVED) { 1414 arq->state = AS_RQ_REMOVED;
1473 arq->state = AS_RQ_DISPATCHED; 1415 if (arq->io_context && arq->io_context->aic)
1474 if (arq->io_context && arq->io_context->aic) 1416 atomic_dec(&arq->io_context->aic->nr_dispatched);
1475 atomic_inc(&arq->io_context->aic->nr_dispatched);
1476 }
1477 } else
1478 WARN_ON(blk_fs_request(rq)
1479 && (!(rq->flags & (REQ_HARDBARRIER|REQ_SOFTBARRIER))) );
1480
1481 /* Stop anticipating - let this request get through */
1482 as_antic_stop(ad);
1483}
1484
1485/*
1486 * requeue the request. The request has not been completed, nor is it a
1487 * new request, so don't touch accounting.
1488 */
1489static void as_requeue_request(request_queue_t *q, struct request *rq)
1490{
1491 as_deactivate_request(q, rq);
1492 list_add(&rq->queuelist, &q->queue_head);
1493}
1494
1495/*
1496 * Account a request that is inserted directly onto the dispatch queue.
1497 * arq->io_context->aic->nr_dispatched should not need to be incremented
1498 * because only new requests should come through here: requeues go through
1499 * our explicit requeue handler.
1500 */
1501static void as_account_queued_request(struct as_data *ad, struct request *rq)
1502{
1503 if (blk_fs_request(rq)) {
1504 struct as_rq *arq = RQ_DATA(rq);
1505 arq->state = AS_RQ_DISPATCHED;
1506 ad->nr_dispatched++;
1507 }
1508} 1417}
1509 1418
1510static void 1419static void as_deactivate_request(request_queue_t *q, struct request *rq)
1511as_insert_request(request_queue_t *q, struct request *rq, int where)
1512{ 1420{
1513 struct as_data *ad = q->elevator->elevator_data;
1514 struct as_rq *arq = RQ_DATA(rq); 1421 struct as_rq *arq = RQ_DATA(rq);
1515 1422
1516 if (arq) { 1423 WARN_ON(arq->state != AS_RQ_REMOVED);
1517 if (arq->state != AS_RQ_PRESCHED) { 1424 arq->state = AS_RQ_DISPATCHED;
1518 printk("arq->state: %d\n", arq->state); 1425 if (arq->io_context && arq->io_context->aic)
1519 WARN_ON(1); 1426 atomic_inc(&arq->io_context->aic->nr_dispatched);
1520 }
1521 arq->state = AS_RQ_NEW;
1522 }
1523
1524 /* barriers must flush the reorder queue */
1525 if (unlikely(rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)
1526 && where == ELEVATOR_INSERT_SORT)) {
1527 WARN_ON(1);
1528 where = ELEVATOR_INSERT_BACK;
1529 }
1530
1531 switch (where) {
1532 case ELEVATOR_INSERT_BACK:
1533 while (ad->next_arq[REQ_SYNC])
1534 as_move_to_dispatch(ad, ad->next_arq[REQ_SYNC]);
1535
1536 while (ad->next_arq[REQ_ASYNC])
1537 as_move_to_dispatch(ad, ad->next_arq[REQ_ASYNC]);
1538
1539 list_add_tail(&rq->queuelist, ad->dispatch);
1540 as_account_queued_request(ad, rq);
1541 as_antic_stop(ad);
1542 break;
1543 case ELEVATOR_INSERT_FRONT:
1544 list_add(&rq->queuelist, ad->dispatch);
1545 as_account_queued_request(ad, rq);
1546 as_antic_stop(ad);
1547 break;
1548 case ELEVATOR_INSERT_SORT:
1549 BUG_ON(!blk_fs_request(rq));
1550 as_add_request(ad, arq);
1551 break;
1552 default:
1553 BUG();
1554 return;
1555 }
1556} 1427}
1557 1428
1558/* 1429/*
@@ -1565,12 +1436,8 @@ static int as_queue_empty(request_queue_t *q)
1565{ 1436{
1566 struct as_data *ad = q->elevator->elevator_data; 1437 struct as_data *ad = q->elevator->elevator_data;
1567 1438
1568 if (!list_empty(&ad->fifo_list[REQ_ASYNC]) 1439 return list_empty(&ad->fifo_list[REQ_ASYNC])
1569 || !list_empty(&ad->fifo_list[REQ_SYNC]) 1440 && list_empty(&ad->fifo_list[REQ_SYNC]);
1570 || !list_empty(ad->dispatch))
1571 return 0;
1572
1573 return 1;
1574} 1441}
1575 1442
1576static struct request * 1443static struct request *
@@ -1608,15 +1475,6 @@ as_merge(request_queue_t *q, struct request **req, struct bio *bio)
1608 int ret; 1475 int ret;
1609 1476
1610 /* 1477 /*
1611 * try last_merge to avoid going to hash
1612 */
1613 ret = elv_try_last_merge(q, bio);
1614 if (ret != ELEVATOR_NO_MERGE) {
1615 __rq = q->last_merge;
1616 goto out_insert;
1617 }
1618
1619 /*
1620 * see if the merge hash can satisfy a back merge 1478 * see if the merge hash can satisfy a back merge
1621 */ 1479 */
1622 __rq = as_find_arq_hash(ad, bio->bi_sector); 1480 __rq = as_find_arq_hash(ad, bio->bi_sector);
@@ -1644,9 +1502,6 @@ as_merge(request_queue_t *q, struct request **req, struct bio *bio)
1644 1502
1645 return ELEVATOR_NO_MERGE; 1503 return ELEVATOR_NO_MERGE;
1646out: 1504out:
1647 if (rq_mergeable(__rq))
1648 q->last_merge = __rq;
1649out_insert:
1650 if (ret) { 1505 if (ret) {
1651 if (rq_mergeable(__rq)) 1506 if (rq_mergeable(__rq))
1652 as_hot_arq_hash(ad, RQ_DATA(__rq)); 1507 as_hot_arq_hash(ad, RQ_DATA(__rq));
@@ -1693,9 +1548,6 @@ static void as_merged_request(request_queue_t *q, struct request *req)
1693 * behind the disk head. We currently don't bother adjusting. 1548 * behind the disk head. We currently don't bother adjusting.
1694 */ 1549 */
1695 } 1550 }
1696
1697 if (arq->on_hash)
1698 q->last_merge = req;
1699} 1551}
1700 1552
1701static void 1553static void
@@ -1763,6 +1615,7 @@ as_merged_requests(request_queue_t *q, struct request *req,
1763 * kill knowledge of next, this one is a goner 1615 * kill knowledge of next, this one is a goner
1764 */ 1616 */
1765 as_remove_queued_request(q, next); 1617 as_remove_queued_request(q, next);
1618 as_put_io_context(anext);
1766 1619
1767 anext->state = AS_RQ_MERGED; 1620 anext->state = AS_RQ_MERGED;
1768} 1621}
@@ -1782,7 +1635,7 @@ static void as_work_handler(void *data)
1782 unsigned long flags; 1635 unsigned long flags;
1783 1636
1784 spin_lock_irqsave(q->queue_lock, flags); 1637 spin_lock_irqsave(q->queue_lock, flags);
1785 if (as_next_request(q)) 1638 if (!as_queue_empty(q))
1786 q->request_fn(q); 1639 q->request_fn(q);
1787 spin_unlock_irqrestore(q->queue_lock, flags); 1640 spin_unlock_irqrestore(q->queue_lock, flags);
1788} 1641}
@@ -1797,7 +1650,9 @@ static void as_put_request(request_queue_t *q, struct request *rq)
1797 return; 1650 return;
1798 } 1651 }
1799 1652
1800 if (arq->state != AS_RQ_POSTSCHED && arq->state != AS_RQ_PRESCHED) { 1653 if (unlikely(arq->state != AS_RQ_POSTSCHED &&
1654 arq->state != AS_RQ_PRESCHED &&
1655 arq->state != AS_RQ_MERGED)) {
1801 printk("arq->state %d\n", arq->state); 1656 printk("arq->state %d\n", arq->state);
1802 WARN_ON(1); 1657 WARN_ON(1);
1803 } 1658 }
@@ -1907,7 +1762,6 @@ static int as_init_queue(request_queue_t *q, elevator_t *e)
1907 INIT_LIST_HEAD(&ad->fifo_list[REQ_ASYNC]); 1762 INIT_LIST_HEAD(&ad->fifo_list[REQ_ASYNC]);
1908 ad->sort_list[REQ_SYNC] = RB_ROOT; 1763 ad->sort_list[REQ_SYNC] = RB_ROOT;
1909 ad->sort_list[REQ_ASYNC] = RB_ROOT; 1764 ad->sort_list[REQ_ASYNC] = RB_ROOT;
1910 ad->dispatch = &q->queue_head;
1911 ad->fifo_expire[REQ_SYNC] = default_read_expire; 1765 ad->fifo_expire[REQ_SYNC] = default_read_expire;
1912 ad->fifo_expire[REQ_ASYNC] = default_write_expire; 1766 ad->fifo_expire[REQ_ASYNC] = default_write_expire;
1913 ad->antic_expire = default_antic_expire; 1767 ad->antic_expire = default_antic_expire;
@@ -2072,10 +1926,9 @@ static struct elevator_type iosched_as = {
2072 .elevator_merge_fn = as_merge, 1926 .elevator_merge_fn = as_merge,
2073 .elevator_merged_fn = as_merged_request, 1927 .elevator_merged_fn = as_merged_request,
2074 .elevator_merge_req_fn = as_merged_requests, 1928 .elevator_merge_req_fn = as_merged_requests,
2075 .elevator_next_req_fn = as_next_request, 1929 .elevator_dispatch_fn = as_dispatch_request,
2076 .elevator_add_req_fn = as_insert_request, 1930 .elevator_add_req_fn = as_add_request,
2077 .elevator_remove_req_fn = as_remove_request, 1931 .elevator_activate_req_fn = as_activate_request,
2078 .elevator_requeue_req_fn = as_requeue_request,
2079 .elevator_deactivate_req_fn = as_deactivate_request, 1932 .elevator_deactivate_req_fn = as_deactivate_request,
2080 .elevator_queue_empty_fn = as_queue_empty, 1933 .elevator_queue_empty_fn = as_queue_empty,
2081 .elevator_completed_req_fn = as_completed_request, 1934 .elevator_completed_req_fn = as_completed_request,
diff --git a/drivers/block/cfq-iosched.c b/drivers/block/cfq-iosched.c
index d3bfe8cfb039..94690e4d41e0 100644
--- a/drivers/block/cfq-iosched.c
+++ b/drivers/block/cfq-iosched.c
@@ -84,7 +84,6 @@ static int cfq_max_depth = 2;
84 (node)->rb_left = NULL; \ 84 (node)->rb_left = NULL; \
85} while (0) 85} while (0)
86#define RB_CLEAR_ROOT(root) ((root)->rb_node = NULL) 86#define RB_CLEAR_ROOT(root) ((root)->rb_node = NULL)
87#define ON_RB(node) ((node)->rb_color != RB_NONE)
88#define rb_entry_crq(node) rb_entry((node), struct cfq_rq, rb_node) 87#define rb_entry_crq(node) rb_entry((node), struct cfq_rq, rb_node)
89#define rq_rb_key(rq) (rq)->sector 88#define rq_rb_key(rq) (rq)->sector
90 89
@@ -271,10 +270,7 @@ CFQ_CFQQ_FNS(expired);
271#undef CFQ_CFQQ_FNS 270#undef CFQ_CFQQ_FNS
272 271
273enum cfq_rq_state_flags { 272enum cfq_rq_state_flags {
274 CFQ_CRQ_FLAG_in_flight = 0, 273 CFQ_CRQ_FLAG_is_sync = 0,
275 CFQ_CRQ_FLAG_in_driver,
276 CFQ_CRQ_FLAG_is_sync,
277 CFQ_CRQ_FLAG_requeued,
278}; 274};
279 275
280#define CFQ_CRQ_FNS(name) \ 276#define CFQ_CRQ_FNS(name) \
@@ -291,14 +287,11 @@ static inline int cfq_crq_##name(const struct cfq_rq *crq) \
291 return (crq->crq_flags & (1 << CFQ_CRQ_FLAG_##name)) != 0; \ 287 return (crq->crq_flags & (1 << CFQ_CRQ_FLAG_##name)) != 0; \
292} 288}
293 289
294CFQ_CRQ_FNS(in_flight);
295CFQ_CRQ_FNS(in_driver);
296CFQ_CRQ_FNS(is_sync); 290CFQ_CRQ_FNS(is_sync);
297CFQ_CRQ_FNS(requeued);
298#undef CFQ_CRQ_FNS 291#undef CFQ_CRQ_FNS
299 292
300static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short); 293static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short);
301static void cfq_dispatch_sort(request_queue_t *, struct cfq_rq *); 294static void cfq_dispatch_insert(request_queue_t *, struct cfq_rq *);
302static void cfq_put_cfqd(struct cfq_data *cfqd); 295static void cfq_put_cfqd(struct cfq_data *cfqd);
303 296
304#define process_sync(tsk) ((tsk)->flags & PF_SYNCWRITE) 297#define process_sync(tsk) ((tsk)->flags & PF_SYNCWRITE)
@@ -311,14 +304,6 @@ static inline void cfq_del_crq_hash(struct cfq_rq *crq)
311 hlist_del_init(&crq->hash); 304 hlist_del_init(&crq->hash);
312} 305}
313 306
314static void cfq_remove_merge_hints(request_queue_t *q, struct cfq_rq *crq)
315{
316 cfq_del_crq_hash(crq);
317
318 if (q->last_merge == crq->request)
319 q->last_merge = NULL;
320}
321
322static inline void cfq_add_crq_hash(struct cfq_data *cfqd, struct cfq_rq *crq) 307static inline void cfq_add_crq_hash(struct cfq_data *cfqd, struct cfq_rq *crq)
323{ 308{
324 const int hash_idx = CFQ_MHASH_FN(rq_hash_key(crq->request)); 309 const int hash_idx = CFQ_MHASH_FN(rq_hash_key(crq->request));
@@ -347,18 +332,13 @@ static struct request *cfq_find_rq_hash(struct cfq_data *cfqd, sector_t offset)
347 return NULL; 332 return NULL;
348} 333}
349 334
350static inline int cfq_pending_requests(struct cfq_data *cfqd)
351{
352 return !list_empty(&cfqd->queue->queue_head) || cfqd->busy_queues;
353}
354
355/* 335/*
356 * scheduler run of queue, if there are requests pending and no one in the 336 * scheduler run of queue, if there are requests pending and no one in the
357 * driver that will restart queueing 337 * driver that will restart queueing
358 */ 338 */
359static inline void cfq_schedule_dispatch(struct cfq_data *cfqd) 339static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
360{ 340{
361 if (!cfqd->rq_in_driver && cfq_pending_requests(cfqd)) 341 if (!cfqd->rq_in_driver && cfqd->busy_queues)
362 kblockd_schedule_work(&cfqd->unplug_work); 342 kblockd_schedule_work(&cfqd->unplug_work);
363} 343}
364 344
@@ -366,7 +346,7 @@ static int cfq_queue_empty(request_queue_t *q)
366{ 346{
367 struct cfq_data *cfqd = q->elevator->elevator_data; 347 struct cfq_data *cfqd = q->elevator->elevator_data;
368 348
369 return !cfq_pending_requests(cfqd); 349 return !cfqd->busy_queues;
370} 350}
371 351
372/* 352/*
@@ -386,11 +366,6 @@ cfq_choose_req(struct cfq_data *cfqd, struct cfq_rq *crq1, struct cfq_rq *crq2)
386 if (crq2 == NULL) 366 if (crq2 == NULL)
387 return crq1; 367 return crq1;
388 368
389 if (cfq_crq_requeued(crq1) && !cfq_crq_requeued(crq2))
390 return crq1;
391 else if (cfq_crq_requeued(crq2) && !cfq_crq_requeued(crq1))
392 return crq2;
393
394 if (cfq_crq_is_sync(crq1) && !cfq_crq_is_sync(crq2)) 369 if (cfq_crq_is_sync(crq1) && !cfq_crq_is_sync(crq2))
395 return crq1; 370 return crq1;
396 else if (cfq_crq_is_sync(crq2) && !cfq_crq_is_sync(crq1)) 371 else if (cfq_crq_is_sync(crq2) && !cfq_crq_is_sync(crq1))
@@ -461,10 +436,7 @@ cfq_find_next_crq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
461 struct cfq_rq *crq_next = NULL, *crq_prev = NULL; 436 struct cfq_rq *crq_next = NULL, *crq_prev = NULL;
462 struct rb_node *rbnext, *rbprev; 437 struct rb_node *rbnext, *rbprev;
463 438
464 rbnext = NULL; 439 if (!(rbnext = rb_next(&last->rb_node))) {
465 if (ON_RB(&last->rb_node))
466 rbnext = rb_next(&last->rb_node);
467 if (!rbnext) {
468 rbnext = rb_first(&cfqq->sort_list); 440 rbnext = rb_first(&cfqq->sort_list);
469 if (rbnext == &last->rb_node) 441 if (rbnext == &last->rb_node)
470 rbnext = NULL; 442 rbnext = NULL;
@@ -545,13 +517,13 @@ static void cfq_resort_rr_list(struct cfq_queue *cfqq, int preempted)
545 * the pending list according to last request service 517 * the pending list according to last request service
546 */ 518 */
547static inline void 519static inline void
548cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq, int requeue) 520cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
549{ 521{
550 BUG_ON(cfq_cfqq_on_rr(cfqq)); 522 BUG_ON(cfq_cfqq_on_rr(cfqq));
551 cfq_mark_cfqq_on_rr(cfqq); 523 cfq_mark_cfqq_on_rr(cfqq);
552 cfqd->busy_queues++; 524 cfqd->busy_queues++;
553 525
554 cfq_resort_rr_list(cfqq, requeue); 526 cfq_resort_rr_list(cfqq, 0);
555} 527}
556 528
557static inline void 529static inline void
@@ -571,22 +543,19 @@ cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
571static inline void cfq_del_crq_rb(struct cfq_rq *crq) 543static inline void cfq_del_crq_rb(struct cfq_rq *crq)
572{ 544{
573 struct cfq_queue *cfqq = crq->cfq_queue; 545 struct cfq_queue *cfqq = crq->cfq_queue;
546 struct cfq_data *cfqd = cfqq->cfqd;
547 const int sync = cfq_crq_is_sync(crq);
574 548
575 if (ON_RB(&crq->rb_node)) { 549 BUG_ON(!cfqq->queued[sync]);
576 struct cfq_data *cfqd = cfqq->cfqd; 550 cfqq->queued[sync]--;
577 const int sync = cfq_crq_is_sync(crq);
578 551
579 BUG_ON(!cfqq->queued[sync]); 552 cfq_update_next_crq(crq);
580 cfqq->queued[sync]--;
581 553
582 cfq_update_next_crq(crq); 554 rb_erase(&crq->rb_node, &cfqq->sort_list);
555 RB_CLEAR_COLOR(&crq->rb_node);
583 556
584 rb_erase(&crq->rb_node, &cfqq->sort_list); 557 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY(&cfqq->sort_list))
585 RB_CLEAR_COLOR(&crq->rb_node); 558 cfq_del_cfqq_rr(cfqd, cfqq);
586
587 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY(&cfqq->sort_list))
588 cfq_del_cfqq_rr(cfqd, cfqq);
589 }
590} 559}
591 560
592static struct cfq_rq * 561static struct cfq_rq *
@@ -627,12 +596,12 @@ static void cfq_add_crq_rb(struct cfq_rq *crq)
627 * if that happens, put the alias on the dispatch list 596 * if that happens, put the alias on the dispatch list
628 */ 597 */
629 while ((__alias = __cfq_add_crq_rb(crq)) != NULL) 598 while ((__alias = __cfq_add_crq_rb(crq)) != NULL)
630 cfq_dispatch_sort(cfqd->queue, __alias); 599 cfq_dispatch_insert(cfqd->queue, __alias);
631 600
632 rb_insert_color(&crq->rb_node, &cfqq->sort_list); 601 rb_insert_color(&crq->rb_node, &cfqq->sort_list);
633 602
634 if (!cfq_cfqq_on_rr(cfqq)) 603 if (!cfq_cfqq_on_rr(cfqq))
635 cfq_add_cfqq_rr(cfqd, cfqq, cfq_crq_requeued(crq)); 604 cfq_add_cfqq_rr(cfqd, cfqq);
636 605
637 /* 606 /*
638 * check if this request is a better next-serve candidate 607 * check if this request is a better next-serve candidate
@@ -643,10 +612,8 @@ static void cfq_add_crq_rb(struct cfq_rq *crq)
643static inline void 612static inline void
644cfq_reposition_crq_rb(struct cfq_queue *cfqq, struct cfq_rq *crq) 613cfq_reposition_crq_rb(struct cfq_queue *cfqq, struct cfq_rq *crq)
645{ 614{
646 if (ON_RB(&crq->rb_node)) { 615 rb_erase(&crq->rb_node, &cfqq->sort_list);
647 rb_erase(&crq->rb_node, &cfqq->sort_list); 616 cfqq->queued[cfq_crq_is_sync(crq)]--;
648 cfqq->queued[cfq_crq_is_sync(crq)]--;
649 }
650 617
651 cfq_add_crq_rb(crq); 618 cfq_add_crq_rb(crq);
652} 619}
@@ -676,49 +643,28 @@ out:
676 return NULL; 643 return NULL;
677} 644}
678 645
679static void cfq_deactivate_request(request_queue_t *q, struct request *rq) 646static void cfq_activate_request(request_queue_t *q, struct request *rq)
680{ 647{
681 struct cfq_data *cfqd = q->elevator->elevator_data; 648 struct cfq_data *cfqd = q->elevator->elevator_data;
682 struct cfq_rq *crq = RQ_DATA(rq);
683
684 if (crq) {
685 struct cfq_queue *cfqq = crq->cfq_queue;
686
687 if (cfq_crq_in_driver(crq)) {
688 cfq_clear_crq_in_driver(crq);
689 WARN_ON(!cfqd->rq_in_driver);
690 cfqd->rq_in_driver--;
691 }
692 if (cfq_crq_in_flight(crq)) {
693 const int sync = cfq_crq_is_sync(crq);
694 649
695 cfq_clear_crq_in_flight(crq); 650 cfqd->rq_in_driver++;
696 WARN_ON(!cfqq->on_dispatch[sync]);
697 cfqq->on_dispatch[sync]--;
698 }
699 cfq_mark_crq_requeued(crq);
700 }
701} 651}
702 652
703/* 653static void cfq_deactivate_request(request_queue_t *q, struct request *rq)
704 * make sure the service time gets corrected on reissue of this request
705 */
706static void cfq_requeue_request(request_queue_t *q, struct request *rq)
707{ 654{
708 cfq_deactivate_request(q, rq); 655 struct cfq_data *cfqd = q->elevator->elevator_data;
709 list_add(&rq->queuelist, &q->queue_head); 656
657 WARN_ON(!cfqd->rq_in_driver);
658 cfqd->rq_in_driver--;
710} 659}
711 660
712static void cfq_remove_request(request_queue_t *q, struct request *rq) 661static void cfq_remove_request(struct request *rq)
713{ 662{
714 struct cfq_rq *crq = RQ_DATA(rq); 663 struct cfq_rq *crq = RQ_DATA(rq);
715 664
716 if (crq) { 665 list_del_init(&rq->queuelist);
717 list_del_init(&rq->queuelist); 666 cfq_del_crq_rb(crq);
718 cfq_del_crq_rb(crq); 667 cfq_del_crq_hash(crq);
719 cfq_remove_merge_hints(q, crq);
720
721 }
722} 668}
723 669
724static int 670static int
@@ -728,12 +674,6 @@ cfq_merge(request_queue_t *q, struct request **req, struct bio *bio)
728 struct request *__rq; 674 struct request *__rq;
729 int ret; 675 int ret;
730 676
731 ret = elv_try_last_merge(q, bio);
732 if (ret != ELEVATOR_NO_MERGE) {
733 __rq = q->last_merge;
734 goto out_insert;
735 }
736
737 __rq = cfq_find_rq_hash(cfqd, bio->bi_sector); 677 __rq = cfq_find_rq_hash(cfqd, bio->bi_sector);
738 if (__rq && elv_rq_merge_ok(__rq, bio)) { 678 if (__rq && elv_rq_merge_ok(__rq, bio)) {
739 ret = ELEVATOR_BACK_MERGE; 679 ret = ELEVATOR_BACK_MERGE;
@@ -748,8 +688,6 @@ cfq_merge(request_queue_t *q, struct request **req, struct bio *bio)
748 688
749 return ELEVATOR_NO_MERGE; 689 return ELEVATOR_NO_MERGE;
750out: 690out:
751 q->last_merge = __rq;
752out_insert:
753 *req = __rq; 691 *req = __rq;
754 return ret; 692 return ret;
755} 693}
@@ -762,14 +700,12 @@ static void cfq_merged_request(request_queue_t *q, struct request *req)
762 cfq_del_crq_hash(crq); 700 cfq_del_crq_hash(crq);
763 cfq_add_crq_hash(cfqd, crq); 701 cfq_add_crq_hash(cfqd, crq);
764 702
765 if (ON_RB(&crq->rb_node) && (rq_rb_key(req) != crq->rb_key)) { 703 if (rq_rb_key(req) != crq->rb_key) {
766 struct cfq_queue *cfqq = crq->cfq_queue; 704 struct cfq_queue *cfqq = crq->cfq_queue;
767 705
768 cfq_update_next_crq(crq); 706 cfq_update_next_crq(crq);
769 cfq_reposition_crq_rb(cfqq, crq); 707 cfq_reposition_crq_rb(cfqq, crq);
770 } 708 }
771
772 q->last_merge = req;
773} 709}
774 710
775static void 711static void
@@ -785,7 +721,7 @@ cfq_merged_requests(request_queue_t *q, struct request *rq,
785 time_before(next->start_time, rq->start_time)) 721 time_before(next->start_time, rq->start_time))
786 list_move(&rq->queuelist, &next->queuelist); 722 list_move(&rq->queuelist, &next->queuelist);
787 723
788 cfq_remove_request(q, next); 724 cfq_remove_request(next);
789} 725}
790 726
791static inline void 727static inline void
@@ -992,53 +928,15 @@ static int cfq_arm_slice_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
992 return 1; 928 return 1;
993} 929}
994 930
995/* 931static void cfq_dispatch_insert(request_queue_t *q, struct cfq_rq *crq)
996 * we dispatch cfqd->cfq_quantum requests in total from the rr_list queues,
997 * this function sector sorts the selected request to minimize seeks. we start
998 * at cfqd->last_sector, not 0.
999 */
1000static void cfq_dispatch_sort(request_queue_t *q, struct cfq_rq *crq)
1001{ 932{
1002 struct cfq_data *cfqd = q->elevator->elevator_data; 933 struct cfq_data *cfqd = q->elevator->elevator_data;
1003 struct cfq_queue *cfqq = crq->cfq_queue; 934 struct cfq_queue *cfqq = crq->cfq_queue;
1004 struct list_head *head = &q->queue_head, *entry = head;
1005 struct request *__rq;
1006 sector_t last;
1007
1008 list_del(&crq->request->queuelist);
1009
1010 last = cfqd->last_sector;
1011 list_for_each_entry_reverse(__rq, head, queuelist) {
1012 struct cfq_rq *__crq = RQ_DATA(__rq);
1013
1014 if (blk_barrier_rq(__rq))
1015 break;
1016 if (!blk_fs_request(__rq))
1017 break;
1018 if (cfq_crq_requeued(__crq))
1019 break;
1020
1021 if (__rq->sector <= crq->request->sector)
1022 break;
1023 if (__rq->sector > last && crq->request->sector < last) {
1024 last = crq->request->sector + crq->request->nr_sectors;
1025 break;
1026 }
1027 entry = &__rq->queuelist;
1028 }
1029
1030 cfqd->last_sector = last;
1031 935
1032 cfqq->next_crq = cfq_find_next_crq(cfqd, cfqq, crq); 936 cfqq->next_crq = cfq_find_next_crq(cfqd, cfqq, crq);
1033 937 cfq_remove_request(crq->request);
1034 cfq_del_crq_rb(crq);
1035 cfq_remove_merge_hints(q, crq);
1036
1037 cfq_mark_crq_in_flight(crq);
1038 cfq_clear_crq_requeued(crq);
1039
1040 cfqq->on_dispatch[cfq_crq_is_sync(crq)]++; 938 cfqq->on_dispatch[cfq_crq_is_sync(crq)]++;
1041 list_add_tail(&crq->request->queuelist, entry); 939 elv_dispatch_sort(q, crq->request);
1042} 940}
1043 941
1044/* 942/*
@@ -1159,7 +1057,7 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq,
1159 /* 1057 /*
1160 * finally, insert request into driver dispatch list 1058 * finally, insert request into driver dispatch list
1161 */ 1059 */
1162 cfq_dispatch_sort(cfqd->queue, crq); 1060 cfq_dispatch_insert(cfqd->queue, crq);
1163 1061
1164 cfqd->dispatch_slice++; 1062 cfqd->dispatch_slice++;
1165 dispatched++; 1063 dispatched++;
@@ -1194,7 +1092,7 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq,
1194} 1092}
1195 1093
1196static int 1094static int
1197cfq_dispatch_requests(request_queue_t *q, int max_dispatch, int force) 1095cfq_dispatch_requests(request_queue_t *q, int force)
1198{ 1096{
1199 struct cfq_data *cfqd = q->elevator->elevator_data; 1097 struct cfq_data *cfqd = q->elevator->elevator_data;
1200 struct cfq_queue *cfqq; 1098 struct cfq_queue *cfqq;
@@ -1204,12 +1102,25 @@ cfq_dispatch_requests(request_queue_t *q, int max_dispatch, int force)
1204 1102
1205 cfqq = cfq_select_queue(cfqd, force); 1103 cfqq = cfq_select_queue(cfqd, force);
1206 if (cfqq) { 1104 if (cfqq) {
1105 int max_dispatch;
1106
1107 /*
1108 * if idle window is disabled, allow queue buildup
1109 */
1110 if (!cfq_cfqq_idle_window(cfqq) &&
1111 cfqd->rq_in_driver >= cfqd->cfq_max_depth)
1112 return 0;
1113
1207 cfq_clear_cfqq_must_dispatch(cfqq); 1114 cfq_clear_cfqq_must_dispatch(cfqq);
1208 cfq_clear_cfqq_wait_request(cfqq); 1115 cfq_clear_cfqq_wait_request(cfqq);
1209 del_timer(&cfqd->idle_slice_timer); 1116 del_timer(&cfqd->idle_slice_timer);
1210 1117
1211 if (cfq_class_idle(cfqq)) 1118 if (!force) {
1212 max_dispatch = 1; 1119 max_dispatch = cfqd->cfq_quantum;
1120 if (cfq_class_idle(cfqq))
1121 max_dispatch = 1;
1122 } else
1123 max_dispatch = INT_MAX;
1213 1124
1214 return __cfq_dispatch_requests(cfqd, cfqq, max_dispatch); 1125 return __cfq_dispatch_requests(cfqd, cfqq, max_dispatch);
1215 } 1126 }
@@ -1217,93 +1128,6 @@ cfq_dispatch_requests(request_queue_t *q, int max_dispatch, int force)
1217 return 0; 1128 return 0;
1218} 1129}
1219 1130
1220static inline void cfq_account_dispatch(struct cfq_rq *crq)
1221{
1222 struct cfq_queue *cfqq = crq->cfq_queue;
1223 struct cfq_data *cfqd = cfqq->cfqd;
1224
1225 if (unlikely(!blk_fs_request(crq->request)))
1226 return;
1227
1228 /*
1229 * accounted bit is necessary since some drivers will call
1230 * elv_next_request() many times for the same request (eg ide)
1231 */
1232 if (cfq_crq_in_driver(crq))
1233 return;
1234
1235 cfq_mark_crq_in_driver(crq);
1236 cfqd->rq_in_driver++;
1237}
1238
1239static inline void
1240cfq_account_completion(struct cfq_queue *cfqq, struct cfq_rq *crq)
1241{
1242 struct cfq_data *cfqd = cfqq->cfqd;
1243 unsigned long now;
1244
1245 if (!cfq_crq_in_driver(crq))
1246 return;
1247
1248 now = jiffies;
1249
1250 WARN_ON(!cfqd->rq_in_driver);
1251 cfqd->rq_in_driver--;
1252
1253 if (!cfq_class_idle(cfqq))
1254 cfqd->last_end_request = now;
1255
1256 if (!cfq_cfqq_dispatched(cfqq)) {
1257 if (cfq_cfqq_on_rr(cfqq)) {
1258 cfqq->service_last = now;
1259 cfq_resort_rr_list(cfqq, 0);
1260 }
1261 if (cfq_cfqq_expired(cfqq)) {
1262 __cfq_slice_expired(cfqd, cfqq, 0);
1263 cfq_schedule_dispatch(cfqd);
1264 }
1265 }
1266
1267 if (cfq_crq_is_sync(crq))
1268 crq->io_context->last_end_request = now;
1269}
1270
1271static struct request *cfq_next_request(request_queue_t *q)
1272{
1273 struct cfq_data *cfqd = q->elevator->elevator_data;
1274 struct request *rq;
1275
1276 if (!list_empty(&q->queue_head)) {
1277 struct cfq_rq *crq;
1278dispatch:
1279 rq = list_entry_rq(q->queue_head.next);
1280
1281 crq = RQ_DATA(rq);
1282 if (crq) {
1283 struct cfq_queue *cfqq = crq->cfq_queue;
1284
1285 /*
1286 * if idle window is disabled, allow queue buildup
1287 */
1288 if (!cfq_crq_in_driver(crq) &&
1289 !cfq_cfqq_idle_window(cfqq) &&
1290 !blk_barrier_rq(rq) &&
1291 cfqd->rq_in_driver >= cfqd->cfq_max_depth)
1292 return NULL;
1293
1294 cfq_remove_merge_hints(q, crq);
1295 cfq_account_dispatch(crq);
1296 }
1297
1298 return rq;
1299 }
1300
1301 if (cfq_dispatch_requests(q, cfqd->cfq_quantum, 0))
1302 goto dispatch;
1303
1304 return NULL;
1305}
1306
1307/* 1131/*
1308 * task holds one reference to the queue, dropped when task exits. each crq 1132 * task holds one reference to the queue, dropped when task exits. each crq
1309 * in-flight on this queue also holds a reference, dropped when crq is freed. 1133 * in-flight on this queue also holds a reference, dropped when crq is freed.
@@ -1816,8 +1640,9 @@ cfq_crq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
1816 } 1640 }
1817} 1641}
1818 1642
1819static void cfq_enqueue(struct cfq_data *cfqd, struct request *rq) 1643static void cfq_insert_request(request_queue_t *q, struct request *rq)
1820{ 1644{
1645 struct cfq_data *cfqd = q->elevator->elevator_data;
1821 struct cfq_rq *crq = RQ_DATA(rq); 1646 struct cfq_rq *crq = RQ_DATA(rq);
1822 struct cfq_queue *cfqq = crq->cfq_queue; 1647 struct cfq_queue *cfqq = crq->cfq_queue;
1823 1648
@@ -1827,66 +1652,43 @@ static void cfq_enqueue(struct cfq_data *cfqd, struct request *rq)
1827 1652
1828 list_add_tail(&rq->queuelist, &cfqq->fifo); 1653 list_add_tail(&rq->queuelist, &cfqq->fifo);
1829 1654
1830 if (rq_mergeable(rq)) { 1655 if (rq_mergeable(rq))
1831 cfq_add_crq_hash(cfqd, crq); 1656 cfq_add_crq_hash(cfqd, crq);
1832 1657
1833 if (!cfqd->queue->last_merge)
1834 cfqd->queue->last_merge = rq;
1835 }
1836
1837 cfq_crq_enqueued(cfqd, cfqq, crq); 1658 cfq_crq_enqueued(cfqd, cfqq, crq);
1838} 1659}
1839 1660
1840static void
1841cfq_insert_request(request_queue_t *q, struct request *rq, int where)
1842{
1843 struct cfq_data *cfqd = q->elevator->elevator_data;
1844
1845 switch (where) {
1846 case ELEVATOR_INSERT_BACK:
1847 while (cfq_dispatch_requests(q, INT_MAX, 1))
1848 ;
1849 list_add_tail(&rq->queuelist, &q->queue_head);
1850 /*
1851 * If we were idling with pending requests on
1852 * inactive cfqqs, force dispatching will
1853 * remove the idle timer and the queue won't
1854 * be kicked by __make_request() afterward.
1855 * Kick it here.
1856 */
1857 cfq_schedule_dispatch(cfqd);
1858 break;
1859 case ELEVATOR_INSERT_FRONT:
1860 list_add(&rq->queuelist, &q->queue_head);
1861 break;
1862 case ELEVATOR_INSERT_SORT:
1863 BUG_ON(!blk_fs_request(rq));
1864 cfq_enqueue(cfqd, rq);
1865 break;
1866 default:
1867 printk("%s: bad insert point %d\n", __FUNCTION__,where);
1868 return;
1869 }
1870}
1871
1872static void cfq_completed_request(request_queue_t *q, struct request *rq) 1661static void cfq_completed_request(request_queue_t *q, struct request *rq)
1873{ 1662{
1874 struct cfq_rq *crq = RQ_DATA(rq); 1663 struct cfq_rq *crq = RQ_DATA(rq);
1875 struct cfq_queue *cfqq; 1664 struct cfq_queue *cfqq = crq->cfq_queue;
1665 struct cfq_data *cfqd = cfqq->cfqd;
1666 const int sync = cfq_crq_is_sync(crq);
1667 unsigned long now;
1876 1668
1877 if (unlikely(!blk_fs_request(rq))) 1669 now = jiffies;
1878 return;
1879 1670
1880 cfqq = crq->cfq_queue; 1671 WARN_ON(!cfqd->rq_in_driver);
1672 WARN_ON(!cfqq->on_dispatch[sync]);
1673 cfqd->rq_in_driver--;
1674 cfqq->on_dispatch[sync]--;
1881 1675
1882 if (cfq_crq_in_flight(crq)) { 1676 if (!cfq_class_idle(cfqq))
1883 const int sync = cfq_crq_is_sync(crq); 1677 cfqd->last_end_request = now;
1884 1678
1885 WARN_ON(!cfqq->on_dispatch[sync]); 1679 if (!cfq_cfqq_dispatched(cfqq)) {
1886 cfqq->on_dispatch[sync]--; 1680 if (cfq_cfqq_on_rr(cfqq)) {
1681 cfqq->service_last = now;
1682 cfq_resort_rr_list(cfqq, 0);
1683 }
1684 if (cfq_cfqq_expired(cfqq)) {
1685 __cfq_slice_expired(cfqd, cfqq, 0);
1686 cfq_schedule_dispatch(cfqd);
1687 }
1887 } 1688 }
1888 1689
1889 cfq_account_completion(cfqq, crq); 1690 if (cfq_crq_is_sync(crq))
1691 crq->io_context->last_end_request = now;
1890} 1692}
1891 1693
1892static struct request * 1694static struct request *
@@ -2118,9 +1920,6 @@ cfq_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
2118 INIT_HLIST_NODE(&crq->hash); 1920 INIT_HLIST_NODE(&crq->hash);
2119 crq->cfq_queue = cfqq; 1921 crq->cfq_queue = cfqq;
2120 crq->io_context = cic; 1922 crq->io_context = cic;
2121 cfq_clear_crq_in_flight(crq);
2122 cfq_clear_crq_in_driver(crq);
2123 cfq_clear_crq_requeued(crq);
2124 1923
2125 if (rw == READ || process_sync(tsk)) 1924 if (rw == READ || process_sync(tsk))
2126 cfq_mark_crq_is_sync(crq); 1925 cfq_mark_crq_is_sync(crq);
@@ -2201,7 +2000,7 @@ static void cfq_idle_slice_timer(unsigned long data)
2201 * only expire and reinvoke request handler, if there are 2000 * only expire and reinvoke request handler, if there are
2202 * other queues with pending requests 2001 * other queues with pending requests
2203 */ 2002 */
2204 if (!cfq_pending_requests(cfqd)) { 2003 if (!cfqd->busy_queues) {
2205 cfqd->idle_slice_timer.expires = min(now + cfqd->cfq_slice_idle, cfqq->slice_end); 2004 cfqd->idle_slice_timer.expires = min(now + cfqd->cfq_slice_idle, cfqq->slice_end);
2206 add_timer(&cfqd->idle_slice_timer); 2005 add_timer(&cfqd->idle_slice_timer);
2207 goto out_cont; 2006 goto out_cont;
@@ -2576,10 +2375,9 @@ static struct elevator_type iosched_cfq = {
2576 .elevator_merge_fn = cfq_merge, 2375 .elevator_merge_fn = cfq_merge,
2577 .elevator_merged_fn = cfq_merged_request, 2376 .elevator_merged_fn = cfq_merged_request,
2578 .elevator_merge_req_fn = cfq_merged_requests, 2377 .elevator_merge_req_fn = cfq_merged_requests,
2579 .elevator_next_req_fn = cfq_next_request, 2378 .elevator_dispatch_fn = cfq_dispatch_requests,
2580 .elevator_add_req_fn = cfq_insert_request, 2379 .elevator_add_req_fn = cfq_insert_request,
2581 .elevator_remove_req_fn = cfq_remove_request, 2380 .elevator_activate_req_fn = cfq_activate_request,
2582 .elevator_requeue_req_fn = cfq_requeue_request,
2583 .elevator_deactivate_req_fn = cfq_deactivate_request, 2381 .elevator_deactivate_req_fn = cfq_deactivate_request,
2584 .elevator_queue_empty_fn = cfq_queue_empty, 2382 .elevator_queue_empty_fn = cfq_queue_empty,
2585 .elevator_completed_req_fn = cfq_completed_request, 2383 .elevator_completed_req_fn = cfq_completed_request,
diff --git a/drivers/block/deadline-iosched.c b/drivers/block/deadline-iosched.c
index 753546ba2262..7929471d7df7 100644
--- a/drivers/block/deadline-iosched.c
+++ b/drivers/block/deadline-iosched.c
@@ -50,7 +50,6 @@ struct deadline_data {
50 * next in sort order. read, write or both are NULL 50 * next in sort order. read, write or both are NULL
51 */ 51 */
52 struct deadline_rq *next_drq[2]; 52 struct deadline_rq *next_drq[2];
53 struct list_head *dispatch; /* driver dispatch queue */
54 struct list_head *hash; /* request hash */ 53 struct list_head *hash; /* request hash */
55 unsigned int batching; /* number of sequential requests made */ 54 unsigned int batching; /* number of sequential requests made */
56 sector_t last_sector; /* head position */ 55 sector_t last_sector; /* head position */
@@ -113,15 +112,6 @@ static inline void deadline_del_drq_hash(struct deadline_rq *drq)
113 __deadline_del_drq_hash(drq); 112 __deadline_del_drq_hash(drq);
114} 113}
115 114
116static void
117deadline_remove_merge_hints(request_queue_t *q, struct deadline_rq *drq)
118{
119 deadline_del_drq_hash(drq);
120
121 if (q->last_merge == drq->request)
122 q->last_merge = NULL;
123}
124
125static inline void 115static inline void
126deadline_add_drq_hash(struct deadline_data *dd, struct deadline_rq *drq) 116deadline_add_drq_hash(struct deadline_data *dd, struct deadline_rq *drq)
127{ 117{
@@ -239,10 +229,9 @@ deadline_del_drq_rb(struct deadline_data *dd, struct deadline_rq *drq)
239 dd->next_drq[data_dir] = rb_entry_drq(rbnext); 229 dd->next_drq[data_dir] = rb_entry_drq(rbnext);
240 } 230 }
241 231
242 if (ON_RB(&drq->rb_node)) { 232 BUG_ON(!ON_RB(&drq->rb_node));
243 rb_erase(&drq->rb_node, DRQ_RB_ROOT(dd, drq)); 233 rb_erase(&drq->rb_node, DRQ_RB_ROOT(dd, drq));
244 RB_CLEAR(&drq->rb_node); 234 RB_CLEAR(&drq->rb_node);
245 }
246} 235}
247 236
248static struct request * 237static struct request *
@@ -286,7 +275,7 @@ deadline_find_first_drq(struct deadline_data *dd, int data_dir)
286/* 275/*
287 * add drq to rbtree and fifo 276 * add drq to rbtree and fifo
288 */ 277 */
289static inline void 278static void
290deadline_add_request(struct request_queue *q, struct request *rq) 279deadline_add_request(struct request_queue *q, struct request *rq)
291{ 280{
292 struct deadline_data *dd = q->elevator->elevator_data; 281 struct deadline_data *dd = q->elevator->elevator_data;
@@ -301,12 +290,8 @@ deadline_add_request(struct request_queue *q, struct request *rq)
301 drq->expires = jiffies + dd->fifo_expire[data_dir]; 290 drq->expires = jiffies + dd->fifo_expire[data_dir];
302 list_add_tail(&drq->fifo, &dd->fifo_list[data_dir]); 291 list_add_tail(&drq->fifo, &dd->fifo_list[data_dir]);
303 292
304 if (rq_mergeable(rq)) { 293 if (rq_mergeable(rq))
305 deadline_add_drq_hash(dd, drq); 294 deadline_add_drq_hash(dd, drq);
306
307 if (!q->last_merge)
308 q->last_merge = rq;
309 }
310} 295}
311 296
312/* 297/*
@@ -315,14 +300,11 @@ deadline_add_request(struct request_queue *q, struct request *rq)
315static void deadline_remove_request(request_queue_t *q, struct request *rq) 300static void deadline_remove_request(request_queue_t *q, struct request *rq)
316{ 301{
317 struct deadline_rq *drq = RQ_DATA(rq); 302 struct deadline_rq *drq = RQ_DATA(rq);
303 struct deadline_data *dd = q->elevator->elevator_data;
318 304
319 if (drq) { 305 list_del_init(&drq->fifo);
320 struct deadline_data *dd = q->elevator->elevator_data; 306 deadline_del_drq_rb(dd, drq);
321 307 deadline_del_drq_hash(drq);
322 list_del_init(&drq->fifo);
323 deadline_remove_merge_hints(q, drq);
324 deadline_del_drq_rb(dd, drq);
325 }
326} 308}
327 309
328static int 310static int
@@ -333,15 +315,6 @@ deadline_merge(request_queue_t *q, struct request **req, struct bio *bio)
333 int ret; 315 int ret;
334 316
335 /* 317 /*
336 * try last_merge to avoid going to hash
337 */
338 ret = elv_try_last_merge(q, bio);
339 if (ret != ELEVATOR_NO_MERGE) {
340 __rq = q->last_merge;
341 goto out_insert;
342 }
343
344 /*
345 * see if the merge hash can satisfy a back merge 318 * see if the merge hash can satisfy a back merge
346 */ 319 */
347 __rq = deadline_find_drq_hash(dd, bio->bi_sector); 320 __rq = deadline_find_drq_hash(dd, bio->bi_sector);
@@ -373,8 +346,6 @@ deadline_merge(request_queue_t *q, struct request **req, struct bio *bio)
373 346
374 return ELEVATOR_NO_MERGE; 347 return ELEVATOR_NO_MERGE;
375out: 348out:
376 q->last_merge = __rq;
377out_insert:
378 if (ret) 349 if (ret)
379 deadline_hot_drq_hash(dd, RQ_DATA(__rq)); 350 deadline_hot_drq_hash(dd, RQ_DATA(__rq));
380 *req = __rq; 351 *req = __rq;
@@ -399,8 +370,6 @@ static void deadline_merged_request(request_queue_t *q, struct request *req)
399 deadline_del_drq_rb(dd, drq); 370 deadline_del_drq_rb(dd, drq);
400 deadline_add_drq_rb(dd, drq); 371 deadline_add_drq_rb(dd, drq);
401 } 372 }
402
403 q->last_merge = req;
404} 373}
405 374
406static void 375static void
@@ -452,7 +421,7 @@ deadline_move_to_dispatch(struct deadline_data *dd, struct deadline_rq *drq)
452 request_queue_t *q = drq->request->q; 421 request_queue_t *q = drq->request->q;
453 422
454 deadline_remove_request(q, drq->request); 423 deadline_remove_request(q, drq->request);
455 list_add_tail(&drq->request->queuelist, dd->dispatch); 424 elv_dispatch_add_tail(q, drq->request);
456} 425}
457 426
458/* 427/*
@@ -502,8 +471,9 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)
502 * deadline_dispatch_requests selects the best request according to 471 * deadline_dispatch_requests selects the best request according to
503 * read/write expire, fifo_batch, etc 472 * read/write expire, fifo_batch, etc
504 */ 473 */
505static int deadline_dispatch_requests(struct deadline_data *dd) 474static int deadline_dispatch_requests(request_queue_t *q, int force)
506{ 475{
476 struct deadline_data *dd = q->elevator->elevator_data;
507 const int reads = !list_empty(&dd->fifo_list[READ]); 477 const int reads = !list_empty(&dd->fifo_list[READ]);
508 const int writes = !list_empty(&dd->fifo_list[WRITE]); 478 const int writes = !list_empty(&dd->fifo_list[WRITE]);
509 struct deadline_rq *drq; 479 struct deadline_rq *drq;
@@ -597,65 +567,12 @@ dispatch_request:
597 return 1; 567 return 1;
598} 568}
599 569
600static struct request *deadline_next_request(request_queue_t *q)
601{
602 struct deadline_data *dd = q->elevator->elevator_data;
603 struct request *rq;
604
605 /*
606 * if there are still requests on the dispatch queue, grab the first one
607 */
608 if (!list_empty(dd->dispatch)) {
609dispatch:
610 rq = list_entry_rq(dd->dispatch->next);
611 return rq;
612 }
613
614 if (deadline_dispatch_requests(dd))
615 goto dispatch;
616
617 return NULL;
618}
619
620static void
621deadline_insert_request(request_queue_t *q, struct request *rq, int where)
622{
623 struct deadline_data *dd = q->elevator->elevator_data;
624
625 /* barriers must flush the reorder queue */
626 if (unlikely(rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)
627 && where == ELEVATOR_INSERT_SORT))
628 where = ELEVATOR_INSERT_BACK;
629
630 switch (where) {
631 case ELEVATOR_INSERT_BACK:
632 while (deadline_dispatch_requests(dd))
633 ;
634 list_add_tail(&rq->queuelist, dd->dispatch);
635 break;
636 case ELEVATOR_INSERT_FRONT:
637 list_add(&rq->queuelist, dd->dispatch);
638 break;
639 case ELEVATOR_INSERT_SORT:
640 BUG_ON(!blk_fs_request(rq));
641 deadline_add_request(q, rq);
642 break;
643 default:
644 printk("%s: bad insert point %d\n", __FUNCTION__,where);
645 return;
646 }
647}
648
649static int deadline_queue_empty(request_queue_t *q) 570static int deadline_queue_empty(request_queue_t *q)
650{ 571{
651 struct deadline_data *dd = q->elevator->elevator_data; 572 struct deadline_data *dd = q->elevator->elevator_data;
652 573
653 if (!list_empty(&dd->fifo_list[WRITE]) 574 return list_empty(&dd->fifo_list[WRITE])
654 || !list_empty(&dd->fifo_list[READ]) 575 && list_empty(&dd->fifo_list[READ]);
655 || !list_empty(dd->dispatch))
656 return 0;
657
658 return 1;
659} 576}
660 577
661static struct request * 578static struct request *
@@ -733,7 +650,6 @@ static int deadline_init_queue(request_queue_t *q, elevator_t *e)
733 INIT_LIST_HEAD(&dd->fifo_list[WRITE]); 650 INIT_LIST_HEAD(&dd->fifo_list[WRITE]);
734 dd->sort_list[READ] = RB_ROOT; 651 dd->sort_list[READ] = RB_ROOT;
735 dd->sort_list[WRITE] = RB_ROOT; 652 dd->sort_list[WRITE] = RB_ROOT;
736 dd->dispatch = &q->queue_head;
737 dd->fifo_expire[READ] = read_expire; 653 dd->fifo_expire[READ] = read_expire;
738 dd->fifo_expire[WRITE] = write_expire; 654 dd->fifo_expire[WRITE] = write_expire;
739 dd->writes_starved = writes_starved; 655 dd->writes_starved = writes_starved;
@@ -748,10 +664,8 @@ static void deadline_put_request(request_queue_t *q, struct request *rq)
748 struct deadline_data *dd = q->elevator->elevator_data; 664 struct deadline_data *dd = q->elevator->elevator_data;
749 struct deadline_rq *drq = RQ_DATA(rq); 665 struct deadline_rq *drq = RQ_DATA(rq);
750 666
751 if (drq) { 667 mempool_free(drq, dd->drq_pool);
752 mempool_free(drq, dd->drq_pool); 668 rq->elevator_private = NULL;
753 rq->elevator_private = NULL;
754 }
755} 669}
756 670
757static int 671static int
@@ -917,9 +831,8 @@ static struct elevator_type iosched_deadline = {
917 .elevator_merge_fn = deadline_merge, 831 .elevator_merge_fn = deadline_merge,
918 .elevator_merged_fn = deadline_merged_request, 832 .elevator_merged_fn = deadline_merged_request,
919 .elevator_merge_req_fn = deadline_merged_requests, 833 .elevator_merge_req_fn = deadline_merged_requests,
920 .elevator_next_req_fn = deadline_next_request, 834 .elevator_dispatch_fn = deadline_dispatch_requests,
921 .elevator_add_req_fn = deadline_insert_request, 835 .elevator_add_req_fn = deadline_add_request,
922 .elevator_remove_req_fn = deadline_remove_request,
923 .elevator_queue_empty_fn = deadline_queue_empty, 836 .elevator_queue_empty_fn = deadline_queue_empty,
924 .elevator_former_req_fn = deadline_former_request, 837 .elevator_former_req_fn = deadline_former_request,
925 .elevator_latter_req_fn = deadline_latter_request, 838 .elevator_latter_req_fn = deadline_latter_request,
diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c
index 4f69d222b183..3b8099ccedff 100644
--- a/drivers/block/elevator.c
+++ b/drivers/block/elevator.c
@@ -83,15 +83,6 @@ inline int elv_try_merge(struct request *__rq, struct bio *bio)
83} 83}
84EXPORT_SYMBOL(elv_try_merge); 84EXPORT_SYMBOL(elv_try_merge);
85 85
86inline int elv_try_last_merge(request_queue_t *q, struct bio *bio)
87{
88 if (q->last_merge)
89 return elv_try_merge(q->last_merge, bio);
90
91 return ELEVATOR_NO_MERGE;
92}
93EXPORT_SYMBOL(elv_try_last_merge);
94
95static struct elevator_type *elevator_find(const char *name) 86static struct elevator_type *elevator_find(const char *name)
96{ 87{
97 struct elevator_type *e = NULL; 88 struct elevator_type *e = NULL;
@@ -143,6 +134,8 @@ static int elevator_attach(request_queue_t *q, struct elevator_type *e,
143 INIT_LIST_HEAD(&q->queue_head); 134 INIT_LIST_HEAD(&q->queue_head);
144 q->last_merge = NULL; 135 q->last_merge = NULL;
145 q->elevator = eq; 136 q->elevator = eq;
137 q->end_sector = 0;
138 q->boundary_rq = NULL;
146 139
147 if (eq->ops->elevator_init_fn) 140 if (eq->ops->elevator_init_fn)
148 ret = eq->ops->elevator_init_fn(q, eq); 141 ret = eq->ops->elevator_init_fn(q, eq);
@@ -225,9 +218,52 @@ void elevator_exit(elevator_t *e)
225 kfree(e); 218 kfree(e);
226} 219}
227 220
221/*
222 * Insert rq into dispatch queue of q. Queue lock must be held on
223 * entry. If sort != 0, rq is sort-inserted; otherwise, rq will be
224 * appended to the dispatch queue. To be used by specific elevators.
225 */
226void elv_dispatch_sort(request_queue_t *q, struct request *rq)
227{
228 sector_t boundary;
229 struct list_head *entry;
230
231 if (q->last_merge == rq)
232 q->last_merge = NULL;
233
234 boundary = q->end_sector;
235
236 list_for_each_prev(entry, &q->queue_head) {
237 struct request *pos = list_entry_rq(entry);
238
239 if (pos->flags & (REQ_SOFTBARRIER|REQ_HARDBARRIER|REQ_STARTED))
240 break;
241 if (rq->sector >= boundary) {
242 if (pos->sector < boundary)
243 continue;
244 } else {
245 if (pos->sector >= boundary)
246 break;
247 }
248 if (rq->sector >= pos->sector)
249 break;
250 }
251
252 list_add(&rq->queuelist, entry);
253}
254
228int elv_merge(request_queue_t *q, struct request **req, struct bio *bio) 255int elv_merge(request_queue_t *q, struct request **req, struct bio *bio)
229{ 256{
230 elevator_t *e = q->elevator; 257 elevator_t *e = q->elevator;
258 int ret;
259
260 if (q->last_merge) {
261 ret = elv_try_merge(q->last_merge, bio);
262 if (ret != ELEVATOR_NO_MERGE) {
263 *req = q->last_merge;
264 return ret;
265 }
266 }
231 267
232 if (e->ops->elevator_merge_fn) 268 if (e->ops->elevator_merge_fn)
233 return e->ops->elevator_merge_fn(q, req, bio); 269 return e->ops->elevator_merge_fn(q, req, bio);
@@ -241,6 +277,8 @@ void elv_merged_request(request_queue_t *q, struct request *rq)
241 277
242 if (e->ops->elevator_merged_fn) 278 if (e->ops->elevator_merged_fn)
243 e->ops->elevator_merged_fn(q, rq); 279 e->ops->elevator_merged_fn(q, rq);
280
281 q->last_merge = rq;
244} 282}
245 283
246void elv_merge_requests(request_queue_t *q, struct request *rq, 284void elv_merge_requests(request_queue_t *q, struct request *rq,
@@ -248,20 +286,13 @@ void elv_merge_requests(request_queue_t *q, struct request *rq,
248{ 286{
249 elevator_t *e = q->elevator; 287 elevator_t *e = q->elevator;
250 288
251 if (q->last_merge == next)
252 q->last_merge = NULL;
253
254 if (e->ops->elevator_merge_req_fn) 289 if (e->ops->elevator_merge_req_fn)
255 e->ops->elevator_merge_req_fn(q, rq, next); 290 e->ops->elevator_merge_req_fn(q, rq, next);
291
292 q->last_merge = rq;
256} 293}
257 294
258/* 295void elv_requeue_request(request_queue_t *q, struct request *rq)
259 * For careful internal use by the block layer. Essentially the same as
260 * a requeue in that it tells the io scheduler that this request is not
261 * active in the driver or hardware anymore, but we don't want the request
262 * added back to the scheduler. Function is not exported.
263 */
264void elv_deactivate_request(request_queue_t *q, struct request *rq)
265{ 296{
266 elevator_t *e = q->elevator; 297 elevator_t *e = q->elevator;
267 298
@@ -269,19 +300,14 @@ void elv_deactivate_request(request_queue_t *q, struct request *rq)
269 * it already went through dequeue, we need to decrement the 300 * it already went through dequeue, we need to decrement the
270 * in_flight count again 301 * in_flight count again
271 */ 302 */
272 if (blk_account_rq(rq)) 303 if (blk_account_rq(rq)) {
273 q->in_flight--; 304 q->in_flight--;
305 if (blk_sorted_rq(rq) && e->ops->elevator_deactivate_req_fn)
306 e->ops->elevator_deactivate_req_fn(q, rq);
307 }
274 308
275 rq->flags &= ~REQ_STARTED; 309 rq->flags &= ~REQ_STARTED;
276 310
277 if (e->ops->elevator_deactivate_req_fn)
278 e->ops->elevator_deactivate_req_fn(q, rq);
279}
280
281void elv_requeue_request(request_queue_t *q, struct request *rq)
282{
283 elv_deactivate_request(q, rq);
284
285 /* 311 /*
286 * if this is the flush, requeue the original instead and drop the flush 312 * if this is the flush, requeue the original instead and drop the flush
287 */ 313 */
@@ -290,55 +316,91 @@ void elv_requeue_request(request_queue_t *q, struct request *rq)
290 rq = rq->end_io_data; 316 rq = rq->end_io_data;
291 } 317 }
292 318
293 /* 319 __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
294 * the request is prepped and may have some resources allocated.
295 * allowing unprepped requests to pass this one may cause resource
296 * deadlock. turn on softbarrier.
297 */
298 rq->flags |= REQ_SOFTBARRIER;
299
300 /*
301 * if iosched has an explicit requeue hook, then use that. otherwise
302 * just put the request at the front of the queue
303 */
304 if (q->elevator->ops->elevator_requeue_req_fn)
305 q->elevator->ops->elevator_requeue_req_fn(q, rq);
306 else
307 __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
308} 320}
309 321
310void __elv_add_request(request_queue_t *q, struct request *rq, int where, 322void __elv_add_request(request_queue_t *q, struct request *rq, int where,
311 int plug) 323 int plug)
312{ 324{
313 /* 325 if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
314 * barriers implicitly indicate back insertion 326 /*
315 */ 327 * barriers implicitly indicate back insertion
316 if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER) && 328 */
317 where == ELEVATOR_INSERT_SORT) 329 if (where == ELEVATOR_INSERT_SORT)
318 where = ELEVATOR_INSERT_BACK; 330 where = ELEVATOR_INSERT_BACK;
331
332 /*
333 * this request is scheduling boundary, update end_sector
334 */
335 if (blk_fs_request(rq)) {
336 q->end_sector = rq_end_sector(rq);
337 q->boundary_rq = rq;
338 }
339 }
319 340
320 if (plug) 341 if (plug)
321 blk_plug_device(q); 342 blk_plug_device(q);
322 343
323 rq->q = q; 344 rq->q = q;
324 345
325 if (!test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)) { 346 if (unlikely(test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags))) {
326 q->elevator->ops->elevator_add_req_fn(q, rq, where);
327
328 if (blk_queue_plugged(q)) {
329 int nrq = q->rq.count[READ] + q->rq.count[WRITE]
330 - q->in_flight;
331
332 if (nrq >= q->unplug_thresh)
333 __generic_unplug_device(q);
334 }
335 } else
336 /* 347 /*
337 * if drain is set, store the request "locally". when the drain 348 * if drain is set, store the request "locally". when the drain
338 * is finished, the requests will be handed ordered to the io 349 * is finished, the requests will be handed ordered to the io
339 * scheduler 350 * scheduler
340 */ 351 */
341 list_add_tail(&rq->queuelist, &q->drain_list); 352 list_add_tail(&rq->queuelist, &q->drain_list);
353 return;
354 }
355
356 switch (where) {
357 case ELEVATOR_INSERT_FRONT:
358 rq->flags |= REQ_SOFTBARRIER;
359
360 list_add(&rq->queuelist, &q->queue_head);
361 break;
362
363 case ELEVATOR_INSERT_BACK:
364 rq->flags |= REQ_SOFTBARRIER;
365
366 while (q->elevator->ops->elevator_dispatch_fn(q, 1))
367 ;
368 list_add_tail(&rq->queuelist, &q->queue_head);
369 /*
370 * We kick the queue here for the following reasons.
371 * - The elevator might have returned NULL previously
372 * to delay requests and returned them now. As the
373 * queue wasn't empty before this request, ll_rw_blk
374 * won't run the queue on return, resulting in hang.
375 * - Usually, back inserted requests won't be merged
376 * with anything. There's no point in delaying queue
377 * processing.
378 */
379 blk_remove_plug(q);
380 q->request_fn(q);
381 break;
382
383 case ELEVATOR_INSERT_SORT:
384 BUG_ON(!blk_fs_request(rq));
385 rq->flags |= REQ_SORTED;
386 q->elevator->ops->elevator_add_req_fn(q, rq);
387 if (q->last_merge == NULL && rq_mergeable(rq))
388 q->last_merge = rq;
389 break;
390
391 default:
392 printk(KERN_ERR "%s: bad insertion point %d\n",
393 __FUNCTION__, where);
394 BUG();
395 }
396
397 if (blk_queue_plugged(q)) {
398 int nrq = q->rq.count[READ] + q->rq.count[WRITE]
399 - q->in_flight;
400
401 if (nrq >= q->unplug_thresh)
402 __generic_unplug_device(q);
403 }
342} 404}
343 405
344void elv_add_request(request_queue_t *q, struct request *rq, int where, 406void elv_add_request(request_queue_t *q, struct request *rq, int where,
@@ -353,13 +415,19 @@ void elv_add_request(request_queue_t *q, struct request *rq, int where,
353 415
354static inline struct request *__elv_next_request(request_queue_t *q) 416static inline struct request *__elv_next_request(request_queue_t *q)
355{ 417{
356 struct request *rq = q->elevator->ops->elevator_next_req_fn(q); 418 struct request *rq;
419
420 if (unlikely(list_empty(&q->queue_head) &&
421 !q->elevator->ops->elevator_dispatch_fn(q, 0)))
422 return NULL;
423
424 rq = list_entry_rq(q->queue_head.next);
357 425
358 /* 426 /*
359 * if this is a barrier write and the device has to issue a 427 * if this is a barrier write and the device has to issue a
360 * flush sequence to support it, check how far we are 428 * flush sequence to support it, check how far we are
361 */ 429 */
362 if (rq && blk_fs_request(rq) && blk_barrier_rq(rq)) { 430 if (blk_fs_request(rq) && blk_barrier_rq(rq)) {
363 BUG_ON(q->ordered == QUEUE_ORDERED_NONE); 431 BUG_ON(q->ordered == QUEUE_ORDERED_NONE);
364 432
365 if (q->ordered == QUEUE_ORDERED_FLUSH && 433 if (q->ordered == QUEUE_ORDERED_FLUSH &&
@@ -376,15 +444,30 @@ struct request *elv_next_request(request_queue_t *q)
376 int ret; 444 int ret;
377 445
378 while ((rq = __elv_next_request(q)) != NULL) { 446 while ((rq = __elv_next_request(q)) != NULL) {
379 /* 447 if (!(rq->flags & REQ_STARTED)) {
380 * just mark as started even if we don't start it, a request 448 elevator_t *e = q->elevator;
381 * that has been delayed should not be passed by new incoming 449
382 * requests 450 /*
383 */ 451 * This is the first time the device driver
384 rq->flags |= REQ_STARTED; 452 * sees this request (possibly after
453 * requeueing). Notify IO scheduler.
454 */
455 if (blk_sorted_rq(rq) &&
456 e->ops->elevator_activate_req_fn)
457 e->ops->elevator_activate_req_fn(q, rq);
385 458
386 if (rq == q->last_merge) 459 /*
387 q->last_merge = NULL; 460 * just mark as started even if we don't start
461 * it, a request that has been delayed should
462 * not be passed by new incoming requests
463 */
464 rq->flags |= REQ_STARTED;
465 }
466
467 if (!q->boundary_rq || q->boundary_rq == rq) {
468 q->end_sector = rq_end_sector(rq);
469 q->boundary_rq = NULL;
470 }
388 471
389 if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn) 472 if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn)
390 break; 473 break;
@@ -396,9 +479,9 @@ struct request *elv_next_request(request_queue_t *q)
396 /* 479 /*
397 * the request may have been (partially) prepped. 480 * the request may have been (partially) prepped.
398 * we need to keep this request in the front to 481 * we need to keep this request in the front to
399 * avoid resource deadlock. turn on softbarrier. 482 * avoid resource deadlock. REQ_STARTED will
483 * prevent other fs requests from passing this one.
400 */ 484 */
401 rq->flags |= REQ_SOFTBARRIER;
402 rq = NULL; 485 rq = NULL;
403 break; 486 break;
404 } else if (ret == BLKPREP_KILL) { 487 } else if (ret == BLKPREP_KILL) {
@@ -421,42 +504,32 @@ struct request *elv_next_request(request_queue_t *q)
421 return rq; 504 return rq;
422} 505}
423 506
424void elv_remove_request(request_queue_t *q, struct request *rq) 507void elv_dequeue_request(request_queue_t *q, struct request *rq)
425{ 508{
426 elevator_t *e = q->elevator; 509 BUG_ON(list_empty(&rq->queuelist));
510
511 list_del_init(&rq->queuelist);
427 512
428 /* 513 /*
429 * the time frame between a request being removed from the lists 514 * the time frame between a request being removed from the lists
430 * and to it is freed is accounted as io that is in progress at 515 * and to it is freed is accounted as io that is in progress at
431 * the driver side. note that we only account requests that the 516 * the driver side.
432 * driver has seen (REQ_STARTED set), to avoid false accounting
433 * for request-request merges
434 */ 517 */
435 if (blk_account_rq(rq)) 518 if (blk_account_rq(rq))
436 q->in_flight++; 519 q->in_flight++;
437
438 /*
439 * the main clearing point for q->last_merge is on retrieval of
440 * request by driver (it calls elv_next_request()), but it _can_
441 * also happen here if a request is added to the queue but later
442 * deleted without ever being given to driver (merged with another
443 * request).
444 */
445 if (rq == q->last_merge)
446 q->last_merge = NULL;
447
448 if (e->ops->elevator_remove_req_fn)
449 e->ops->elevator_remove_req_fn(q, rq);
450} 520}
451 521
452int elv_queue_empty(request_queue_t *q) 522int elv_queue_empty(request_queue_t *q)
453{ 523{
454 elevator_t *e = q->elevator; 524 elevator_t *e = q->elevator;
455 525
526 if (!list_empty(&q->queue_head))
527 return 0;
528
456 if (e->ops->elevator_queue_empty_fn) 529 if (e->ops->elevator_queue_empty_fn)
457 return e->ops->elevator_queue_empty_fn(q); 530 return e->ops->elevator_queue_empty_fn(q);
458 531
459 return list_empty(&q->queue_head); 532 return 1;
460} 533}
461 534
462struct request *elv_latter_request(request_queue_t *q, struct request *rq) 535struct request *elv_latter_request(request_queue_t *q, struct request *rq)
@@ -528,11 +601,11 @@ void elv_completed_request(request_queue_t *q, struct request *rq)
528 /* 601 /*
529 * request is released from the driver, io must be done 602 * request is released from the driver, io must be done
530 */ 603 */
531 if (blk_account_rq(rq)) 604 if (blk_account_rq(rq)) {
532 q->in_flight--; 605 q->in_flight--;
533 606 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
534 if (e->ops->elevator_completed_req_fn) 607 e->ops->elevator_completed_req_fn(q, rq);
535 e->ops->elevator_completed_req_fn(q, rq); 608 }
536} 609}
537 610
538int elv_register_queue(struct request_queue *q) 611int elv_register_queue(struct request_queue *q)
@@ -705,11 +778,12 @@ ssize_t elv_iosched_show(request_queue_t *q, char *name)
705 return len; 778 return len;
706} 779}
707 780
781EXPORT_SYMBOL(elv_dispatch_sort);
708EXPORT_SYMBOL(elv_add_request); 782EXPORT_SYMBOL(elv_add_request);
709EXPORT_SYMBOL(__elv_add_request); 783EXPORT_SYMBOL(__elv_add_request);
710EXPORT_SYMBOL(elv_requeue_request); 784EXPORT_SYMBOL(elv_requeue_request);
711EXPORT_SYMBOL(elv_next_request); 785EXPORT_SYMBOL(elv_next_request);
712EXPORT_SYMBOL(elv_remove_request); 786EXPORT_SYMBOL(elv_dequeue_request);
713EXPORT_SYMBOL(elv_queue_empty); 787EXPORT_SYMBOL(elv_queue_empty);
714EXPORT_SYMBOL(elv_completed_request); 788EXPORT_SYMBOL(elv_completed_request);
715EXPORT_SYMBOL(elevator_exit); 789EXPORT_SYMBOL(elevator_exit);
diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index ac31ea170058..c4f186223bc5 100644
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -353,6 +353,8 @@ static void blk_pre_flush_end_io(struct request *flush_rq)
353 struct request *rq = flush_rq->end_io_data; 353 struct request *rq = flush_rq->end_io_data;
354 request_queue_t *q = rq->q; 354 request_queue_t *q = rq->q;
355 355
356 elv_completed_request(q, flush_rq);
357
356 rq->flags |= REQ_BAR_PREFLUSH; 358 rq->flags |= REQ_BAR_PREFLUSH;
357 359
358 if (!flush_rq->errors) 360 if (!flush_rq->errors)
@@ -369,6 +371,8 @@ static void blk_post_flush_end_io(struct request *flush_rq)
369 struct request *rq = flush_rq->end_io_data; 371 struct request *rq = flush_rq->end_io_data;
370 request_queue_t *q = rq->q; 372 request_queue_t *q = rq->q;
371 373
374 elv_completed_request(q, flush_rq);
375
372 rq->flags |= REQ_BAR_POSTFLUSH; 376 rq->flags |= REQ_BAR_POSTFLUSH;
373 377
374 q->end_flush_fn(q, flush_rq); 378 q->end_flush_fn(q, flush_rq);
@@ -408,8 +412,6 @@ struct request *blk_start_pre_flush(request_queue_t *q, struct request *rq)
408 if (!list_empty(&rq->queuelist)) 412 if (!list_empty(&rq->queuelist))
409 blkdev_dequeue_request(rq); 413 blkdev_dequeue_request(rq);
410 414
411 elv_deactivate_request(q, rq);
412
413 flush_rq->end_io_data = rq; 415 flush_rq->end_io_data = rq;
414 flush_rq->end_io = blk_pre_flush_end_io; 416 flush_rq->end_io = blk_pre_flush_end_io;
415 417
@@ -1040,6 +1042,7 @@ EXPORT_SYMBOL(blk_queue_invalidate_tags);
1040static char *rq_flags[] = { 1042static char *rq_flags[] = {
1041 "REQ_RW", 1043 "REQ_RW",
1042 "REQ_FAILFAST", 1044 "REQ_FAILFAST",
1045 "REQ_SORTED",
1043 "REQ_SOFTBARRIER", 1046 "REQ_SOFTBARRIER",
1044 "REQ_HARDBARRIER", 1047 "REQ_HARDBARRIER",
1045 "REQ_CMD", 1048 "REQ_CMD",
@@ -2456,6 +2459,8 @@ static void __blk_put_request(request_queue_t *q, struct request *req)
2456 if (unlikely(--req->ref_count)) 2459 if (unlikely(--req->ref_count))
2457 return; 2460 return;
2458 2461
2462 elv_completed_request(q, req);
2463
2459 req->rq_status = RQ_INACTIVE; 2464 req->rq_status = RQ_INACTIVE;
2460 req->rl = NULL; 2465 req->rl = NULL;
2461 2466
@@ -2466,8 +2471,6 @@ static void __blk_put_request(request_queue_t *q, struct request *req)
2466 if (rl) { 2471 if (rl) {
2467 int rw = rq_data_dir(req); 2472 int rw = rq_data_dir(req);
2468 2473
2469 elv_completed_request(q, req);
2470
2471 BUG_ON(!list_empty(&req->queuelist)); 2474 BUG_ON(!list_empty(&req->queuelist));
2472 2475
2473 blk_free_request(q, req); 2476 blk_free_request(q, req);
@@ -2477,14 +2480,14 @@ static void __blk_put_request(request_queue_t *q, struct request *req)
2477 2480
2478void blk_put_request(struct request *req) 2481void blk_put_request(struct request *req)
2479{ 2482{
2483 unsigned long flags;
2484 request_queue_t *q = req->q;
2485
2480 /* 2486 /*
2481 * if req->rl isn't set, this request didnt originate from the 2487 * Gee, IDE calls in w/ NULL q. Fix IDE and remove the
2482 * block layer, so it's safe to just disregard it 2488 * following if (q) test.
2483 */ 2489 */
2484 if (req->rl) { 2490 if (q) {
2485 unsigned long flags;
2486 request_queue_t *q = req->q;
2487
2488 spin_lock_irqsave(q->queue_lock, flags); 2491 spin_lock_irqsave(q->queue_lock, flags);
2489 __blk_put_request(q, req); 2492 __blk_put_request(q, req);
2490 spin_unlock_irqrestore(q->queue_lock, flags); 2493 spin_unlock_irqrestore(q->queue_lock, flags);
diff --git a/drivers/block/noop-iosched.c b/drivers/block/noop-iosched.c
index b1730b62c37e..f56b8edb06e4 100644
--- a/drivers/block/noop-iosched.c
+++ b/drivers/block/noop-iosched.c
@@ -7,57 +7,19 @@
7#include <linux/module.h> 7#include <linux/module.h>
8#include <linux/init.h> 8#include <linux/init.h>
9 9
10/* 10static void elevator_noop_add_request(request_queue_t *q, struct request *rq)
11 * See if we can find a request that this buffer can be coalesced with.
12 */
13static int elevator_noop_merge(request_queue_t *q, struct request **req,
14 struct bio *bio)
15{
16 int ret;
17
18 ret = elv_try_last_merge(q, bio);
19 if (ret != ELEVATOR_NO_MERGE)
20 *req = q->last_merge;
21
22 return ret;
23}
24
25static void elevator_noop_merge_requests(request_queue_t *q, struct request *req,
26 struct request *next)
27{
28 list_del_init(&next->queuelist);
29}
30
31static void elevator_noop_add_request(request_queue_t *q, struct request *rq,
32 int where)
33{ 11{
34 if (where == ELEVATOR_INSERT_FRONT) 12 elv_dispatch_add_tail(q, rq);
35 list_add(&rq->queuelist, &q->queue_head);
36 else
37 list_add_tail(&rq->queuelist, &q->queue_head);
38
39 /*
40 * new merges must not precede this barrier
41 */
42 if (rq->flags & REQ_HARDBARRIER)
43 q->last_merge = NULL;
44 else if (!q->last_merge)
45 q->last_merge = rq;
46} 13}
47 14
48static struct request *elevator_noop_next_request(request_queue_t *q) 15static int elevator_noop_dispatch(request_queue_t *q, int force)
49{ 16{
50 if (!list_empty(&q->queue_head)) 17 return 0;
51 return list_entry_rq(q->queue_head.next);
52
53 return NULL;
54} 18}
55 19
56static struct elevator_type elevator_noop = { 20static struct elevator_type elevator_noop = {
57 .ops = { 21 .ops = {
58 .elevator_merge_fn = elevator_noop_merge, 22 .elevator_dispatch_fn = elevator_noop_dispatch,
59 .elevator_merge_req_fn = elevator_noop_merge_requests,
60 .elevator_next_req_fn = elevator_noop_next_request,
61 .elevator_add_req_fn = elevator_noop_add_request, 23 .elevator_add_req_fn = elevator_noop_add_request,
62 }, 24 },
63 .elevator_name = "noop", 25 .elevator_name = "noop",
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-10 06:18:55 -0500