aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/i386/kernel/apic.c4
-rw-r--r--arch/i386/kernel/cpu/mcheck/p4.c2
-rw-r--r--arch/i386/kernel/irq.c3
-rw-r--r--arch/i386/kernel/process.c53
-rw-r--r--arch/i386/kernel/smp.c2
-rw-r--r--include/asm-i386/idle.h14
-rw-r--r--include/asm-i386/processor.h8
7 files changed, 85 insertions, 1 deletions
diff --git a/arch/i386/kernel/apic.c b/arch/i386/kernel/apic.c
index 629c5ed94260..f4159e0a7ae9 100644
--- a/arch/i386/kernel/apic.c
+++ b/arch/i386/kernel/apic.c
@@ -36,6 +36,7 @@
36#include <asm/hpet.h> 36#include <asm/hpet.h>
37#include <asm/i8253.h> 37#include <asm/i8253.h>
38#include <asm/nmi.h> 38#include <asm/nmi.h>
39#include <asm/idle.h>
39 40
40#include <mach_apic.h> 41#include <mach_apic.h>
41#include <mach_apicdef.h> 42#include <mach_apicdef.h>
@@ -1255,6 +1256,7 @@ fastcall void smp_apic_timer_interrupt(struct pt_regs *regs)
1255 * Besides, if we don't timer interrupts ignore the global 1256 * Besides, if we don't timer interrupts ignore the global
1256 * interrupt lock, which is the WrongThing (tm) to do. 1257 * interrupt lock, which is the WrongThing (tm) to do.
1257 */ 1258 */
1259 exit_idle();
1258 irq_enter(); 1260 irq_enter();
1259 smp_local_timer_interrupt(); 1261 smp_local_timer_interrupt();
1260 irq_exit(); 1262 irq_exit();
@@ -1305,6 +1307,7 @@ fastcall void smp_spurious_interrupt(struct pt_regs *regs)
1305{ 1307{
1306 unsigned long v; 1308 unsigned long v;
1307 1309
1310 exit_idle();
1308 irq_enter(); 1311 irq_enter();
1309 /* 1312 /*
1310 * Check if this really is a spurious interrupt and ACK it 1313 * Check if this really is a spurious interrupt and ACK it
@@ -1329,6 +1332,7 @@ fastcall void smp_error_interrupt(struct pt_regs *regs)
1329{ 1332{
1330 unsigned long v, v1; 1333 unsigned long v, v1;
1331 1334
1335 exit_idle();
1332 irq_enter(); 1336 irq_enter();
1333 /* First tickle the hardware, only then report what went on. -- REW */ 1337 /* First tickle the hardware, only then report what went on. -- REW */
1334 v = apic_read(APIC_ESR); 1338 v = apic_read(APIC_ESR);
diff --git a/arch/i386/kernel/cpu/mcheck/p4.c b/arch/i386/kernel/cpu/mcheck/p4.c
index 504434a46011..8359c19d3a23 100644
--- a/arch/i386/kernel/cpu/mcheck/p4.c
+++ b/arch/i386/kernel/cpu/mcheck/p4.c
@@ -12,6 +12,7 @@
12#include <asm/system.h> 12#include <asm/system.h>
13#include <asm/msr.h> 13#include <asm/msr.h>
14#include <asm/apic.h> 14#include <asm/apic.h>
15#include <asm/idle.h>
15 16
16#include <asm/therm_throt.h> 17#include <asm/therm_throt.h>
17 18
@@ -59,6 +60,7 @@ static void (*vendor_thermal_interrupt)(struct pt_regs *regs) = unexpected_therm
59 60
60fastcall void smp_thermal_interrupt(struct pt_regs *regs) 61fastcall void smp_thermal_interrupt(struct pt_regs *regs)
61{ 62{
63 exit_idle();
62 irq_enter(); 64 irq_enter();
63 vendor_thermal_interrupt(regs); 65 vendor_thermal_interrupt(regs);
64 irq_exit(); 66 irq_exit();
diff --git a/arch/i386/kernel/irq.c b/arch/i386/kernel/irq.c
index 3201d421090a..5785d84103a6 100644
--- a/arch/i386/kernel/irq.c
+++ b/arch/i386/kernel/irq.c
@@ -19,6 +19,8 @@
19#include <linux/cpu.h> 19#include <linux/cpu.h>
20#include <linux/delay.h> 20#include <linux/delay.h>
21 21
22#include <asm/idle.h>
23
22DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp; 24DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp;
23EXPORT_PER_CPU_SYMBOL(irq_stat); 25EXPORT_PER_CPU_SYMBOL(irq_stat);
24 26
@@ -61,6 +63,7 @@ fastcall unsigned int do_IRQ(struct pt_regs *regs)
61 union irq_ctx *curctx, *irqctx; 63 union irq_ctx *curctx, *irqctx;
62 u32 *isp; 64 u32 *isp;
63#endif 65#endif
66 exit_idle();
64 67
65 if (unlikely((unsigned)irq >= NR_IRQS)) { 68 if (unlikely((unsigned)irq >= NR_IRQS)) {
66 printk(KERN_EMERG "%s: cannot handle IRQ %d\n", 69 printk(KERN_EMERG "%s: cannot handle IRQ %d\n",
diff --git a/arch/i386/kernel/process.c b/arch/i386/kernel/process.c
index 05be77413351..7845d480c293 100644
--- a/arch/i386/kernel/process.c
+++ b/arch/i386/kernel/process.c
@@ -48,6 +48,7 @@
48#include <asm/i387.h> 48#include <asm/i387.h>
49#include <asm/desc.h> 49#include <asm/desc.h>
50#include <asm/vm86.h> 50#include <asm/vm86.h>
51#include <asm/idle.h>
51#ifdef CONFIG_MATH_EMULATION 52#ifdef CONFIG_MATH_EMULATION
52#include <asm/math_emu.h> 53#include <asm/math_emu.h>
53#endif 54#endif
@@ -80,6 +81,42 @@ void (*pm_idle)(void);
80EXPORT_SYMBOL(pm_idle); 81EXPORT_SYMBOL(pm_idle);
81static DEFINE_PER_CPU(unsigned int, cpu_idle_state); 82static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
82 83
84static ATOMIC_NOTIFIER_HEAD(idle_notifier);
85
86void idle_notifier_register(struct notifier_block *n)
87{
88 atomic_notifier_chain_register(&idle_notifier, n);
89}
90
91void idle_notifier_unregister(struct notifier_block *n)
92{
93 atomic_notifier_chain_unregister(&idle_notifier, n);
94}
95
96static DEFINE_PER_CPU(volatile unsigned long, idle_state);
97
98void enter_idle(void)
99{
100 /* needs to be atomic w.r.t. interrupts, not against other CPUs */
101 __set_bit(0, &__get_cpu_var(idle_state));
102 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
103}
104
105static void __exit_idle(void)
106{
107 /* needs to be atomic w.r.t. interrupts, not against other CPUs */
108 if (__test_and_clear_bit(0, &__get_cpu_var(idle_state)) == 0)
109 return;
110 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
111}
112
113void exit_idle(void)
114{
115 if (current->pid)
116 return;
117 __exit_idle();
118}
119
83void disable_hlt(void) 120void disable_hlt(void)
84{ 121{
85 hlt_counter++; 122 hlt_counter++;
@@ -130,6 +167,7 @@ EXPORT_SYMBOL(default_idle);
130 */ 167 */
131static void poll_idle (void) 168static void poll_idle (void)
132{ 169{
170 local_irq_enable();
133 cpu_relax(); 171 cpu_relax();
134} 172}
135 173
@@ -189,7 +227,16 @@ void cpu_idle(void)
189 play_dead(); 227 play_dead();
190 228
191 __get_cpu_var(irq_stat).idle_timestamp = jiffies; 229 __get_cpu_var(irq_stat).idle_timestamp = jiffies;
230
231 /*
232 * Idle routines should keep interrupts disabled
233 * from here on, until they go to idle.
234 * Otherwise, idle callbacks can misfire.
235 */
236 local_irq_disable();
237 enter_idle();
192 idle(); 238 idle();
239 __exit_idle();
193 } 240 }
194 preempt_enable_no_resched(); 241 preempt_enable_no_resched();
195 schedule(); 242 schedule();
@@ -243,7 +290,11 @@ void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
243 __monitor((void *)&current_thread_info()->flags, 0, 0); 290 __monitor((void *)&current_thread_info()->flags, 0, 0);
244 smp_mb(); 291 smp_mb();
245 if (!need_resched()) 292 if (!need_resched())
246 __mwait(eax, ecx); 293 __sti_mwait(eax, ecx);
294 else
295 local_irq_enable();
296 } else {
297 local_irq_enable();
247 } 298 }
248} 299}
249 300
diff --git a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c
index 5285aff8367f..ffc4f65c5189 100644
--- a/arch/i386/kernel/smp.c
+++ b/arch/i386/kernel/smp.c
@@ -23,6 +23,7 @@
23 23
24#include <asm/mtrr.h> 24#include <asm/mtrr.h>
25#include <asm/tlbflush.h> 25#include <asm/tlbflush.h>
26#include <asm/idle.h>
26#include <mach_apic.h> 27#include <mach_apic.h>
27 28
28/* 29/*
@@ -624,6 +625,7 @@ fastcall void smp_call_function_interrupt(struct pt_regs *regs)
624 /* 625 /*
625 * At this point the info structure may be out of scope unless wait==1 626 * At this point the info structure may be out of scope unless wait==1
626 */ 627 */
628 exit_idle();
627 irq_enter(); 629 irq_enter();
628 (*func)(info); 630 (*func)(info);
629 irq_exit(); 631 irq_exit();
diff --git a/include/asm-i386/idle.h b/include/asm-i386/idle.h
new file mode 100644
index 000000000000..87ab93911199
--- /dev/null
+++ b/include/asm-i386/idle.h
@@ -0,0 +1,14 @@
1#ifndef _ASM_I386_IDLE_H
2#define _ASM_I386_IDLE_H 1
3
4#define IDLE_START 1
5#define IDLE_END 2
6
7struct notifier_block;
8void idle_notifier_register(struct notifier_block *n);
9void idle_notifier_unregister(struct notifier_block *n);
10
11void exit_idle(void);
12void enter_idle(void);
13
14#endif
diff --git a/include/asm-i386/processor.h b/include/asm-i386/processor.h
index 11bf899de8aa..edfbe46a5e13 100644
--- a/include/asm-i386/processor.h
+++ b/include/asm-i386/processor.h
@@ -257,6 +257,14 @@ static inline void __mwait(unsigned long eax, unsigned long ecx)
257 : :"a" (eax), "c" (ecx)); 257 : :"a" (eax), "c" (ecx));
258} 258}
259 259
260static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
261{
262 /* "mwait %eax,%ecx;" */
263 asm volatile(
264 "sti; .byte 0x0f,0x01,0xc9;"
265 : :"a" (eax), "c" (ecx));
266}
267
260extern void mwait_idle_with_hints(unsigned long eax, unsigned long ecx); 268extern void mwait_idle_with_hints(unsigned long eax, unsigned long ecx);
261 269
262/* from system description table in BIOS. Mostly for MCA use, but 270/* from system description table in BIOS. Mostly for MCA use, but