sched: Provide iowait counters

For counting how long an application has been waiting for
(disk) IO, there currently is only the HZ sample driven
information available, while for all other counters in this
class, a high resolution version is available via
CONFIG_SCHEDSTATS.

In order to make an improved bootchart tool possible, we also
need a higher resolution version of the iowait time.

This patch below adds this scheduler statistic to the kernel.

Signed-off-by: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <4A64B813.1080506@linux.intel.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

3 files changed, 13 insertions, 0 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 6244d24cafc1..38d05a89e0f2 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -6754,7 +6754,9 @@ void __sched io_schedule(void)
 
         delayacct_blkio_start();
         atomic_inc(&rq->nr_iowait);
+        current->in_iowait = 1;
         schedule();
+        current->in_iowait = 0;
         atomic_dec(&rq->nr_iowait);
         delayacct_blkio_end();
 }
@@ -6767,7 +6769,9 @@ long __sched io_schedule_timeout(long timeout)
 
         delayacct_blkio_start();
         atomic_inc(&rq->nr_iowait);
+        current->in_iowait = 1;
         ret = schedule_timeout(timeout);
+        current->in_iowait = 0;
         atomic_dec(&rq->nr_iowait);
         delayacct_blkio_end();
         return ret;
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 70c7e0b79946..5ddbd0891267 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -409,6 +409,8 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
         PN(se.wait_max);
         PN(se.wait_sum);
         P(se.wait_count);
+        PN(se.iowait_sum);
+        P(se.iowait_count);
         P(sched_info.bkl_count);
         P(se.nr_migrations);
         P(se.nr_migrations_cold);
@@ -479,6 +481,8 @@ void proc_sched_set_task(struct task_struct *p)
         p->se.wait_max                          = 0;
         p->se.wait_sum                          = 0;
         p->se.wait_count                        = 0;
+        p->se.iowait_sum                        = 0;
+        p->se.iowait_count                      = 0;
         p->se.sleep_max                         = 0;
         p->se.sum_sleep_runtime                 = 0;
         p->se.block_max                         = 0;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 342000b31ad6..471fa281f5e0 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -652,6 +652,11 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 se->sum_sleep_runtime += delta;
 
                 if (tsk) {
+                        if (tsk->in_iowait) {
+                                se->iowait_sum += delta;
+                                se->iowait_count++;
+                        }
+
                         /*
                          * Blocking time is in units of nanosecs, so shift by
                          * 20 to get a milliseconds-range estimation of the