Merge branch 'master'

4 files changed, 167 insertions, 50 deletions

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index ecacca9c87..2b64f5852b 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -861,8 +861,8 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
          * store what was left of this slice, if the queue idled out
          * or was preempted
          */
-        if (time_after(now, cfqq->slice_end))
-                cfqq->slice_left = now - cfqq->slice_end;
+        if (time_after(cfqq->slice_end, now))
+                cfqq->slice_left = cfqq->slice_end - now;
         else
                 cfqq->slice_left = 0;
 
@@ -999,7 +999,7 @@ cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 /*
  * get next queue for service
  */
-static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd, int force)
+static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
 {
         unsigned long now = jiffies;
         struct cfq_queue *cfqq;
@@ -1023,7 +1023,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd, int force)
          */
         if (!RB_EMPTY(&cfqq->sort_list))
                 goto keep_queue;
-        else if (!force && cfq_cfqq_class_sync(cfqq) &&
+        else if (cfq_cfqq_class_sync(cfqq) &&
                  time_before(now, cfqq->slice_end)) {
                 if (cfq_arm_slice_timer(cfqd, cfqq))
                         return NULL;
@@ -1092,6 +1092,42 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 }
 
 static int
+cfq_forced_dispatch_cfqqs(struct list_head *list)
+{
+        int dispatched = 0;
+        struct cfq_queue *cfqq, *next;
+        struct cfq_rq *crq;
+
+        list_for_each_entry_safe(cfqq, next, list, cfq_list) {
+                while ((crq = cfqq->next_crq)) {
+                        cfq_dispatch_insert(cfqq->cfqd->queue, crq);
+                        dispatched++;
+                }
+                BUG_ON(!list_empty(&cfqq->fifo));
+        }
+        return dispatched;
+}
+
+static int
+cfq_forced_dispatch(struct cfq_data *cfqd)
+{
+        int i, dispatched = 0;
+
+        for (i = 0; i < CFQ_PRIO_LISTS; i++)
+                dispatched += cfq_forced_dispatch_cfqqs(&cfqd->rr_list[i]);
+
+        dispatched += cfq_forced_dispatch_cfqqs(&cfqd->busy_rr);
+        dispatched += cfq_forced_dispatch_cfqqs(&cfqd->cur_rr);
+        dispatched += cfq_forced_dispatch_cfqqs(&cfqd->idle_rr);
+
+        cfq_slice_expired(cfqd, 0);
+
+        BUG_ON(cfqd->busy_queues);
+
+        return dispatched;
+}
+
+static int
 cfq_dispatch_requests(request_queue_t *q, int force)
 {
         struct cfq_data *cfqd = q->elevator->elevator_data;
@@ -1100,7 +1136,10 @@ cfq_dispatch_requests(request_queue_t *q, int force)
         if (!cfqd->busy_queues)
                 return 0;
 
-        cfqq = cfq_select_queue(cfqd, force);
+        if (unlikely(force))
+                return cfq_forced_dispatch(cfqd);
+
+        cfqq = cfq_select_queue(cfqd);
         if (cfqq) {
                 int max_dispatch;
 
@@ -1115,12 +1154,9 @@ cfq_dispatch_requests(request_queue_t *q, int force)
                 cfq_clear_cfqq_wait_request(cfqq);
                 del_timer(&cfqd->idle_slice_timer);
 
-                if (!force) {
-                        max_dispatch = cfqd->cfq_quantum;
-                        if (cfq_class_idle(cfqq))
-                                max_dispatch = 1;
-                } else
-                        max_dispatch = INT_MAX;
+                max_dispatch = cfqd->cfq_quantum;
+                if (cfq_class_idle(cfqq))
+                        max_dispatch = 1;
 
                 return __cfq_dispatch_requests(cfqd, cfqq, max_dispatch);
         }
diff --git a/block/elevator.c b/block/elevator.c
index d4a49a3df8..e4c58827bb 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -155,9 +155,10 @@ static void elevator_setup_default(void)
         /*
          * If the given scheduler is not available, fall back to no-op.
          */
-        if (!(e = elevator_find(chosen_elevator)))
+        if ((e = elevator_find(chosen_elevator)))
+                elevator_put(e);
+        else
                 strcpy(chosen_elevator, "noop");
-        elevator_put(e);
 }
 
 static int __init elevator_setup(char *str)
@@ -190,14 +191,14 @@ int elevator_init(request_queue_t *q, char *name)
 
         eq = kmalloc(sizeof(struct elevator_queue), GFP_KERNEL);
         if (!eq) {
-                elevator_put(e->elevator_type);
+                elevator_put(e);
                 return -ENOMEM;
         }
 
         ret = elevator_attach(q, e, eq);
         if (ret) {
                 kfree(eq);
-                elevator_put(e->elevator_type);
+                elevator_put(e);
         }
 
         return ret;
@@ -225,6 +226,7 @@ void elv_dispatch_sort(request_queue_t *q, struct request *rq)
 
         if (q->last_merge == rq)
                 q->last_merge = NULL;
+        q->nr_sorted--;
 
         boundary = q->end_sector;
 
@@ -283,6 +285,7 @@ void elv_merge_requests(request_queue_t *q, struct request *rq,
 
         if (e->ops->elevator_merge_req_fn)
                 e->ops->elevator_merge_req_fn(q, rq, next);
+        q->nr_sorted--;
 
         q->last_merge = rq;
 }
@@ -314,6 +317,20 @@ void elv_requeue_request(request_queue_t *q, struct request *rq)
         __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
 }
 
+static void elv_drain_elevator(request_queue_t *q)
+{
+        static int printed;
+        while (q->elevator->ops->elevator_dispatch_fn(q, 1))
+                ;
+        if (q->nr_sorted == 0)
+                return;
+        if (printed++ < 10) {
+                printk(KERN_ERR "%s: forced dispatching is broken "
+                       "(nr_sorted=%u), please report this\n",
+                       q->elevator->elevator_type->elevator_name, q->nr_sorted);
+        }
+}
+
 void __elv_add_request(request_queue_t *q, struct request *rq, int where,
                        int plug)
 {
@@ -348,9 +365,7 @@ void __elv_add_request(request_queue_t *q, struct request *rq, int where,
 
         case ELEVATOR_INSERT_BACK:
                 rq->flags |= REQ_SOFTBARRIER;
-
-                while (q->elevator->ops->elevator_dispatch_fn(q, 1))
-                        ;
+                elv_drain_elevator(q);
                 list_add_tail(&rq->queuelist, &q->queue_head);
                 /*
                  * We kick the queue here for the following reasons.
@@ -369,6 +384,7 @@ void __elv_add_request(request_queue_t *q, struct request *rq, int where,
         case ELEVATOR_INSERT_SORT:
                 BUG_ON(!blk_fs_request(rq));
                 rq->flags |= REQ_SORTED;
+                q->nr_sorted++;
                 if (q->last_merge == NULL && rq_mergeable(rq))
                         q->last_merge = rq;
                 /*
@@ -525,33 +541,19 @@ int elv_queue_empty(request_queue_t *q)
 
 struct request *elv_latter_request(request_queue_t *q, struct request *rq)
 {
-        struct list_head *next;
-
         elevator_t *e = q->elevator;
 
         if (e->ops->elevator_latter_req_fn)
                 return e->ops->elevator_latter_req_fn(q, rq);
-
-        next = rq->queuelist.next;
-        if (next != &q->queue_head && next != &rq->queuelist)
-                return list_entry_rq(next);
-
         return NULL;
 }
 
 struct request *elv_former_request(request_queue_t *q, struct request *rq)
 {
-        struct list_head *prev;
-
         elevator_t *e = q->elevator;
 
         if (e->ops->elevator_former_req_fn)
                 return e->ops->elevator_former_req_fn(q, rq);
-
-        prev = rq->queuelist.prev;
-        if (prev != &q->queue_head && prev != &rq->queuelist)
-                return list_entry_rq(prev);
-
         return NULL;
 }
 
@@ -691,13 +693,15 @@ static void elevator_switch(request_queue_t *q, struct elevator_type *new_e)
 
         set_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
 
-        while (q->elevator->ops->elevator_dispatch_fn(q, 1))
-                ;
+        elv_drain_elevator(q);
 
         while (q->rq.elvpriv) {
+                blk_remove_plug(q);
+                q->request_fn(q);
                 spin_unlock_irq(q->queue_lock);
                 msleep(10);
                 spin_lock_irq(q->queue_lock);
+                elv_drain_elevator(q);
         }
 
         spin_unlock_irq(q->queue_lock);
@@ -744,13 +748,15 @@ error:
 ssize_t elv_iosched_store(request_queue_t *q, const char *name, size_t count)
 {
         char elevator_name[ELV_NAME_MAX];
+        size_t len;
         struct elevator_type *e;
 
-        memset(elevator_name, 0, sizeof(elevator_name));
-        strncpy(elevator_name, name, sizeof(elevator_name));
+        elevator_name[sizeof(elevator_name) - 1] = '\0';
+        strncpy(elevator_name, name, sizeof(elevator_name) - 1);
+        len = strlen(elevator_name);
 
-        if (elevator_name[strlen(elevator_name) - 1] == '\n')
-                elevator_name[strlen(elevator_name) - 1] = '\0';
+        if (len && elevator_name[len - 1] == '\n')
+                elevator_name[len - 1] = '\0';
 
         e = elevator_get(elevator_name);
         if (!e) {
diff --git a/block/genhd.c b/block/genhd.c
index 54aec4a1ae..f04609d553 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -391,12 +391,14 @@ static ssize_t disk_stats_read(struct gendisk * disk, char *page)
                 "%8u %8u %8llu %8u "
                 "%8u %8u %8u"
                 "\n",
-                disk_stat_read(disk, ios[0]), disk_stat_read(disk, merges[0]),
-                (unsigned long long)disk_stat_read(disk, sectors[0]),
-                jiffies_to_msecs(disk_stat_read(disk, ticks[0])),
-                disk_stat_read(disk, ios[1]), disk_stat_read(disk, merges[1]),
-                (unsigned long long)disk_stat_read(disk, sectors[1]),
-                jiffies_to_msecs(disk_stat_read(disk, ticks[1])),
+                disk_stat_read(disk, ios[READ]),
+                disk_stat_read(disk, merges[READ]),
+                (unsigned long long)disk_stat_read(disk, sectors[READ]),
+                jiffies_to_msecs(disk_stat_read(disk, ticks[READ])),
+                disk_stat_read(disk, ios[WRITE]),
+                disk_stat_read(disk, merges[WRITE]),
+                (unsigned long long)disk_stat_read(disk, sectors[WRITE]),
+                jiffies_to_msecs(disk_stat_read(disk, ticks[WRITE])),
                 disk->in_flight,
                 jiffies_to_msecs(disk_stat_read(disk, io_ticks)),
                 jiffies_to_msecs(disk_stat_read(disk, time_in_queue)));
diff --git a/block/noop-iosched.c b/block/noop-iosched.c
index e54f006e7e..f370e4a7fe 100644
--- a/block/noop-iosched.c
+++ b/block/noop-iosched.c
@@ -7,21 +7,94 @@
 #include <linux/module.h>
 #include <linux/init.h>
 
-static void elevator_noop_add_request(request_queue_t *q, struct request *rq)
+struct noop_data {
+        struct list_head queue;
+};
+
+static void noop_merged_requests(request_queue_t *q, struct request *rq,
+                                 struct request *next)
+{
+        list_del_init(&next->queuelist);
+}
+
+static int noop_dispatch(request_queue_t *q, int force)
+{
+        struct noop_data *nd = q->elevator->elevator_data;
+
+        if (!list_empty(&nd->queue)) {
+                struct request *rq;
+                rq = list_entry(nd->queue.next, struct request, queuelist);
+                list_del_init(&rq->queuelist);
+                elv_dispatch_sort(q, rq);
+                return 1;
+        }
+        return 0;
+}
+
+static void noop_add_request(request_queue_t *q, struct request *rq)
+{
+        struct noop_data *nd = q->elevator->elevator_data;
+
+        list_add_tail(&rq->queuelist, &nd->queue);
+}
+
+static int noop_queue_empty(request_queue_t *q)
 {
-        rq->flags |= REQ_NOMERGE;
-        elv_dispatch_add_tail(q, rq);
+        struct noop_data *nd = q->elevator->elevator_data;
+
+        return list_empty(&nd->queue);
+}
+
+static struct request *
+noop_former_request(request_queue_t *q, struct request *rq)
+{
+        struct noop_data *nd = q->elevator->elevator_data;
+
+        if (rq->queuelist.prev == &nd->queue)
+                return NULL;
+        return list_entry(rq->queuelist.prev, struct request, queuelist);
+}
+
+static struct request *
+noop_latter_request(request_queue_t *q, struct request *rq)
+{
+        struct noop_data *nd = q->elevator->elevator_data;
+
+        if (rq->queuelist.next == &nd->queue)
+                return NULL;
+        return list_entry(rq->queuelist.next, struct request, queuelist);
 }
 
-static int elevator_noop_dispatch(request_queue_t *q, int force)
+static int noop_init_queue(request_queue_t *q, elevator_t *e)
 {
+        struct noop_data *nd;
+
+        nd = kmalloc(sizeof(*nd), GFP_KERNEL);
+        if (!nd)
+                return -ENOMEM;
+        INIT_LIST_HEAD(&nd->queue);
+        e->elevator_data = nd;
         return 0;
 }
 
+static void noop_exit_queue(elevator_t *e)
+{
+        struct noop_data *nd = e->elevator_data;
+
+        BUG_ON(!list_empty(&nd->queue));
+        kfree(nd);
+}
+
 static struct elevator_type elevator_noop = {
         .ops = {
-                .elevator_dispatch_fn           = elevator_noop_dispatch,
-                .elevator_add_req_fn            = elevator_noop_add_request,
+                .elevator_merge_req_fn          = noop_merged_requests,
+                .elevator_dispatch_fn           = noop_dispatch,
+                .elevator_add_req_fn            = noop_add_request,
+                .elevator_queue_empty_fn        = noop_queue_empty,
+                .elevator_former_req_fn         = noop_former_request,
+                .elevator_latter_req_fn         = noop_latter_request,
+                .elevator_init_fn               = noop_init_queue,
+                .elevator_exit_fn               = noop_exit_queue,
         },
         .elevator_name = "noop",
         .elevator_owner = THIS_MODULE,