block: fix inconsistency in I/O stat accounting code

This forces in_flight to be zero when turning off or on the I/O stat
accounting and stops updating I/O stats in attempt_merge() when
accounting is turned off.

Signed-off-by: Jerome Marchand <jmarchan@redhat.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

6 files changed, 33 insertions, 26 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 25572802dac2..3688abff2430 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -64,12 +64,11 @@ static struct workqueue_struct *kblockd_workqueue;
 
 static void drive_stat_acct(struct request *rq, int new_io)
 {
-        struct gendisk *disk = rq->rq_disk;
         struct hd_struct *part;
         int rw = rq_data_dir(rq);
         int cpu;
 
-        if (!blk_fs_request(rq) || !disk || !blk_do_io_stat(disk->queue))
+        if (!blk_fs_request(rq) || !blk_do_io_stat(rq))
                 return;
 
         cpu = part_stat_lock();
@@ -1675,9 +1674,7 @@ EXPORT_SYMBOL(blkdev_dequeue_request);
 
 static void blk_account_io_completion(struct request *req, unsigned int bytes)
 {
-        struct gendisk *disk = req->rq_disk;
-
-        if (!disk || !blk_do_io_stat(disk->queue))
+        if (!blk_do_io_stat(req))
                 return;
 
         if (blk_fs_request(req)) {
@@ -1694,9 +1691,7 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes)
 
 static void blk_account_io_done(struct request *req)
 {
-        struct gendisk *disk = req->rq_disk;
-
-        if (!disk || !blk_do_io_stat(disk->queue))
+        if (!blk_do_io_stat(req))
                 return;
 
         /*
@@ -1711,7 +1706,7 @@ static void blk_account_io_done(struct request *req)
                 int cpu;
 
                 cpu = part_stat_lock();
-                part = disk_map_sector_rcu(disk, req->sector);
+                part = disk_map_sector_rcu(req->rq_disk, req->sector);
 
                 part_stat_inc(cpu, part, ios[rw]);
                 part_stat_add(cpu, part, ticks[rw], duration);
diff --git a/block/blk-merge.c b/block/blk-merge.c
index e39cb24b7679..63760ca3da0f 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -338,6 +338,22 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
         return 1;
 }
 
+static void blk_account_io_merge(struct request *req)
+{
+        if (blk_do_io_stat(req)) {
+                struct hd_struct *part;
+                int cpu;
+
+                cpu = part_stat_lock();
+                part = disk_map_sector_rcu(req->rq_disk, req->sector);
+
+                part_round_stats(cpu, part);
+                part_dec_in_flight(part);
+
+                part_stat_unlock();
+        }
+}
+
 /*
  * Has to be called with the request spinlock acquired
  */
@@ -386,18 +402,7 @@ static int attempt_merge(struct request_queue *q, struct request *req,
 
         elv_merge_requests(q, req, next);
 
-        if (req->rq_disk) {
-                struct hd_struct *part;
-                int cpu;
-
-                cpu = part_stat_lock();
-                part = disk_map_sector_rcu(req->rq_disk, req->sector);
-
-                part_round_stats(cpu, part);
-                part_dec_in_flight(part);
-
-                part_stat_unlock();
-        }
+        blk_account_io_merge(req);
 
         req->ioprio = ioprio_best(req->ioprio, next->ioprio);
         if (blk_rq_cpu_valid(next))
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 3ff9bba3379a..73f36beff5cd 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -209,10 +209,14 @@ static ssize_t queue_iostats_store(struct request_queue *q, const char *page,
         ssize_t ret = queue_var_store(&stats, page, count);
 
         spin_lock_irq(q->queue_lock);
+        elv_quisce_start(q);
+
         if (stats)
                 queue_flag_set(QUEUE_FLAG_IO_STAT, q);
         else
                 queue_flag_clear(QUEUE_FLAG_IO_STAT, q);
+
+        elv_quisce_end(q);
         spin_unlock_irq(q->queue_lock);
 
         return ret;
diff --git a/block/blk.h b/block/blk.h
index 22043c2886c7..24fcaeeaf620 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -112,12 +112,14 @@ static inline int blk_cpu_to_group(int cpu)
 #endif
 }
 
-static inline int blk_do_io_stat(struct request_queue *q)
+static inline int blk_do_io_stat(struct request *rq)
 {
-        if (q)
-                return blk_queue_io_stat(q);
+        struct gendisk *disk = rq->rq_disk;
 
-        return 0;
+        if (!disk || !disk->queue)
+                return 0;
+
+        return blk_queue_io_stat(disk->queue) && (rq->cmd_flags & REQ_ELVPRIV);
 }
 
 #endif
diff --git a/block/elevator.c b/block/elevator.c
index c6744913ff4a..fb81bcc14a8c 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -573,7 +573,7 @@ void elv_requeue_request(struct request_queue *q, struct request *rq)
         elv_insert(q, rq, ELEVATOR_INSERT_REQUEUE);
 }
 
-static void elv_drain_elevator(struct request_queue *q)
+void elv_drain_elevator(struct request_queue *q)
 {
         static int printed;
         while (q->elevator->ops->elevator_dispatch_fn(q, 1))
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index 7a204256b155..c59b769f62b0 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -116,6 +116,7 @@ extern void elv_abort_queue(struct request_queue *);
 extern void elv_completed_request(struct request_queue *, struct request *);
 extern int elv_set_request(struct request_queue *, struct request *, gfp_t);
 extern void elv_put_request(struct request_queue *, struct request *);
+extern void elv_drain_elevator(struct request_queue *);
 
 /*
  * io scheduler registration