aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorFrederic Weisbecker <fweisbec@gmail.com>2012-10-05 22:07:19 -0400
committerFrederic Weisbecker <fweisbec@gmail.com>2012-10-29 16:31:32 -0400
commitfa5058f3b63153e0147ef65bcdb3a4ee63581346 (patch)
treed9d88284e22953fc8a175b334f1ead0c455e4cd1
parentb080935c8638e08134629d0a9ebdf35669bec14d (diff)
cputime: Specialize irq vtime hooks
With CONFIG_VIRT_CPU_ACCOUNTING, when vtime_account() is called in irq entry/exit, we perform a check on the context: if we are interrupting the idle task we account the pending cputime to idle, otherwise account to system time or its sub-areas: tsk->stime, hardirq time, softirq time, ... However this check for idle only concerns the hardirq entry and softirq entry: * Hardirq may directly interrupt the idle task, in which case we need to flush the pending CPU time to idle. * The idle task may be directly interrupted by a softirq if it calls local_bh_enable(). There is probably no such call in any idle task but we need to cover every case. Ksoftirqd is not concerned because the idle time is flushed on context switch and softirq in the end of hardirq have the idle time already flushed from the hardirq entry. In the other cases we always account to system/irq time: * On hardirq exit we account the time to hardirq time. * On softirq exit we account the time to softirq time. To optimize this and avoid the indirect call to vtime_account() and the checks it performs, specialize the vtime irq APIs and only perform the check on irq entry. Irq exit can directly call vtime_account_system(). CONFIG_IRQ_TIME_ACCOUNTING behaviour doesn't change and directly maps to its own vtime_account() implementation. One may want to take benefits from the new APIs to optimize irq time accounting as well in the future. Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Ingo Molnar <mingo@kernel.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Paul Gortmaker <paul.gortmaker@windriver.com>
-rw-r--r--include/linux/hardirq.h4
-rw-r--r--include/linux/vtime.h25
-rw-r--r--kernel/softirq.c6
3 files changed, 30 insertions, 5 deletions
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index b083a475423d..624ef3f45c8e 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -153,7 +153,7 @@ extern void rcu_nmi_exit(void);
153 */ 153 */
154#define __irq_enter() \ 154#define __irq_enter() \
155 do { \ 155 do { \
156 vtime_account(current); \ 156 vtime_account_irq_enter(current); \
157 add_preempt_count(HARDIRQ_OFFSET); \ 157 add_preempt_count(HARDIRQ_OFFSET); \
158 trace_hardirq_enter(); \ 158 trace_hardirq_enter(); \
159 } while (0) 159 } while (0)
@@ -169,7 +169,7 @@ extern void irq_enter(void);
169#define __irq_exit() \ 169#define __irq_exit() \
170 do { \ 170 do { \
171 trace_hardirq_exit(); \ 171 trace_hardirq_exit(); \
172 vtime_account(current); \ 172 vtime_account_irq_exit(current); \
173 sub_preempt_count(HARDIRQ_OFFSET); \ 173 sub_preempt_count(HARDIRQ_OFFSET); \
174 } while (0) 174 } while (0)
175 175
diff --git a/include/linux/vtime.h b/include/linux/vtime.h
index b9fc4f9ab470..c35c02223da8 100644
--- a/include/linux/vtime.h
+++ b/include/linux/vtime.h
@@ -21,4 +21,29 @@ static inline void vtime_account(struct task_struct *tsk)
21extern void vtime_account(struct task_struct *tsk); 21extern void vtime_account(struct task_struct *tsk);
22#endif 22#endif
23 23
24static inline void vtime_account_irq_enter(struct task_struct *tsk)
25{
26 /*
27 * Hardirq can interrupt idle task anytime. So we need vtime_account()
28 * that performs the idle check in CONFIG_VIRT_CPU_ACCOUNTING.
29 * Softirq can also interrupt idle task directly if it calls
30 * local_bh_enable(). Such case probably don't exist but we never know.
31 * Ksoftirqd is not concerned because idle time is flushed on context
32 * switch. Softirqs in the end of hardirqs are also not a problem because
33 * the idle time is flushed on hardirq time already.
34 */
35 vtime_account(tsk);
36}
37
38static inline void vtime_account_irq_exit(struct task_struct *tsk)
39{
40#ifdef CONFIG_VIRT_CPU_ACCOUNTING
41 /* On hard|softirq exit we always account to hard|softirq cputime */
42 __vtime_account_system(tsk);
43#endif
44#ifdef CONFIG_IRQ_TIME_ACCOUNTING
45 vtime_account(tsk);
46#endif
47}
48
24#endif /* _LINUX_KERNEL_VTIME_H */ 49#endif /* _LINUX_KERNEL_VTIME_H */
diff --git a/kernel/softirq.c b/kernel/softirq.c
index cc96bdc0c2c9..ed567babe789 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -221,7 +221,7 @@ asmlinkage void __do_softirq(void)
221 current->flags &= ~PF_MEMALLOC; 221 current->flags &= ~PF_MEMALLOC;
222 222
223 pending = local_softirq_pending(); 223 pending = local_softirq_pending();
224 vtime_account(current); 224 vtime_account_irq_enter(current);
225 225
226 __local_bh_disable((unsigned long)__builtin_return_address(0), 226 __local_bh_disable((unsigned long)__builtin_return_address(0),
227 SOFTIRQ_OFFSET); 227 SOFTIRQ_OFFSET);
@@ -272,7 +272,7 @@ restart:
272 272
273 lockdep_softirq_exit(); 273 lockdep_softirq_exit();
274 274
275 vtime_account(current); 275 vtime_account_irq_exit(current);
276 __local_bh_enable(SOFTIRQ_OFFSET); 276 __local_bh_enable(SOFTIRQ_OFFSET);
277 tsk_restore_flags(current, old_flags, PF_MEMALLOC); 277 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
278} 278}
@@ -341,7 +341,7 @@ static inline void invoke_softirq(void)
341 */ 341 */
342void irq_exit(void) 342void irq_exit(void)
343{ 343{
344 vtime_account(current); 344 vtime_account_irq_exit(current);
345 trace_hardirq_exit(); 345 trace_hardirq_exit();
346 sub_preempt_count(IRQ_EXIT_OFFSET); 346 sub_preempt_count(IRQ_EXIT_OFFSET);
347 if (!in_interrupt() && local_softirq_pending()) 347 if (!in_interrupt() && local_softirq_pending())