aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/alpha/include/asm/Kbuild1
-rw-r--r--arch/arc/include/asm/Kbuild1
-rw-r--r--arch/arm/include/asm/Kbuild1
-rw-r--r--arch/arm64/include/asm/Kbuild1
-rw-r--r--arch/avr32/include/asm/Kbuild1
-rw-r--r--arch/blackfin/include/asm/Kbuild1
-rw-r--r--arch/c6x/include/asm/Kbuild1
-rw-r--r--arch/cris/include/asm/Kbuild1
-rw-r--r--arch/frv/include/asm/Kbuild1
-rw-r--r--arch/h8300/include/asm/Kbuild1
-rw-r--r--arch/hexagon/include/asm/Kbuild1
-rw-r--r--arch/ia64/include/asm/Kbuild1
-rw-r--r--arch/m32r/include/asm/Kbuild1
-rw-r--r--arch/m68k/include/asm/Kbuild1
-rw-r--r--arch/metag/include/asm/Kbuild1
-rw-r--r--arch/metag/include/asm/topology.h2
-rw-r--r--arch/microblaze/include/asm/Kbuild1
-rw-r--r--arch/mips/include/asm/Kbuild1
-rw-r--r--arch/mips/kernel/rtlx.c19
-rw-r--r--arch/mips/mm/init.c5
-rw-r--r--arch/mn10300/include/asm/Kbuild1
-rw-r--r--arch/openrisc/include/asm/Kbuild1
-rw-r--r--arch/parisc/include/asm/Kbuild1
-rw-r--r--arch/powerpc/include/asm/Kbuild1
-rw-r--r--arch/s390/include/asm/Kbuild1
-rw-r--r--arch/score/include/asm/Kbuild1
-rw-r--r--arch/sh/include/asm/Kbuild1
-rw-r--r--arch/sparc/include/asm/Kbuild1
-rw-r--r--arch/tile/include/asm/Kbuild1
-rw-r--r--arch/um/include/asm/Kbuild1
-rw-r--r--arch/unicore32/include/asm/Kbuild1
-rw-r--r--arch/x86/include/asm/atomic.h29
-rw-r--r--arch/x86/include/asm/atomic64_64.h28
-rw-r--r--arch/x86/include/asm/bitops.h24
-rw-r--r--arch/x86/include/asm/calling.h50
-rw-r--r--arch/x86/include/asm/local.h28
-rw-r--r--arch/x86/include/asm/preempt.h100
-rw-r--r--arch/x86/include/asm/rmwcc.h41
-rw-r--r--arch/x86/include/asm/thread_info.h5
-rw-r--r--arch/x86/kernel/Makefile2
-rw-r--r--arch/x86/kernel/asm-offsets.c1
-rw-r--r--arch/x86/kernel/cpu/common.c5
-rw-r--r--arch/x86/kernel/entry_32.S7
-rw-r--r--arch/x86/kernel/entry_64.S4
-rw-r--r--arch/x86/kernel/i386_ksyms_32.c7
-rw-r--r--arch/x86/kernel/irq_32.c4
-rw-r--r--arch/x86/kernel/preempt.S25
-rw-r--r--arch/x86/kernel/process.c6
-rw-r--r--arch/x86/kernel/process_32.c8
-rw-r--r--arch/x86/kernel/process_64.c8
-rw-r--r--arch/x86/kernel/traps.c4
-rw-r--r--arch/x86/kernel/x8664_ksyms_64.c7
-rw-r--r--arch/xtensa/include/asm/Kbuild1
53 files changed, 322 insertions, 126 deletions
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild
index a6e85f448c1c..f01fb505ad52 100644
--- a/arch/alpha/include/asm/Kbuild
+++ b/arch/alpha/include/asm/Kbuild
@@ -3,3 +3,4 @@ generic-y += clkdev.h
3 3
4generic-y += exec.h 4generic-y += exec.h
5generic-y += trace_clock.h 5generic-y += trace_clock.h
6generic-y += preempt.h
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild
index d8dd660898b9..5943f7f9d325 100644
--- a/arch/arc/include/asm/Kbuild
+++ b/arch/arc/include/asm/Kbuild
@@ -46,3 +46,4 @@ generic-y += ucontext.h
46generic-y += user.h 46generic-y += user.h
47generic-y += vga.h 47generic-y += vga.h
48generic-y += xor.h 48generic-y += xor.h
49generic-y += preempt.h
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index 59ceae8f3c95..1a7024b41351 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -32,3 +32,4 @@ generic-y += termios.h
32generic-y += timex.h 32generic-y += timex.h
33generic-y += trace_clock.h 33generic-y += trace_clock.h
34generic-y += unaligned.h 34generic-y += unaligned.h
35generic-y += preempt.h
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild
index 79a642d199f2..519f89f5b6a3 100644
--- a/arch/arm64/include/asm/Kbuild
+++ b/arch/arm64/include/asm/Kbuild
@@ -50,3 +50,4 @@ generic-y += unaligned.h
50generic-y += user.h 50generic-y += user.h
51generic-y += vga.h 51generic-y += vga.h
52generic-y += xor.h 52generic-y += xor.h
53generic-y += preempt.h
diff --git a/arch/avr32/include/asm/Kbuild b/arch/avr32/include/asm/Kbuild
index fd7980743890..658001b52400 100644
--- a/arch/avr32/include/asm/Kbuild
+++ b/arch/avr32/include/asm/Kbuild
@@ -7,6 +7,7 @@ generic-y += div64.h
7generic-y += emergency-restart.h 7generic-y += emergency-restart.h
8generic-y += exec.h 8generic-y += exec.h
9generic-y += futex.h 9generic-y += futex.h
10generic-y += preempt.h
10generic-y += irq_regs.h 11generic-y += irq_regs.h
11generic-y += param.h 12generic-y += param.h
12generic-y += local.h 13generic-y += local.h
diff --git a/arch/blackfin/include/asm/Kbuild b/arch/blackfin/include/asm/Kbuild
index 127826f8a375..f2b43474b0e2 100644
--- a/arch/blackfin/include/asm/Kbuild
+++ b/arch/blackfin/include/asm/Kbuild
@@ -44,3 +44,4 @@ generic-y += ucontext.h
44generic-y += unaligned.h 44generic-y += unaligned.h
45generic-y += user.h 45generic-y += user.h
46generic-y += xor.h 46generic-y += xor.h
47generic-y += preempt.h
diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild
index e49f918531ad..fc0b3c356027 100644
--- a/arch/c6x/include/asm/Kbuild
+++ b/arch/c6x/include/asm/Kbuild
@@ -56,3 +56,4 @@ generic-y += ucontext.h
56generic-y += user.h 56generic-y += user.h
57generic-y += vga.h 57generic-y += vga.h
58generic-y += xor.h 58generic-y += xor.h
59generic-y += preempt.h
diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild
index c8325455520e..b06caf649a95 100644
--- a/arch/cris/include/asm/Kbuild
+++ b/arch/cris/include/asm/Kbuild
@@ -11,3 +11,4 @@ generic-y += module.h
11generic-y += trace_clock.h 11generic-y += trace_clock.h
12generic-y += vga.h 12generic-y += vga.h
13generic-y += xor.h 13generic-y += xor.h
14generic-y += preempt.h
diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild
index c5d767028306..74742dc6a3da 100644
--- a/arch/frv/include/asm/Kbuild
+++ b/arch/frv/include/asm/Kbuild
@@ -2,3 +2,4 @@
2generic-y += clkdev.h 2generic-y += clkdev.h
3generic-y += exec.h 3generic-y += exec.h
4generic-y += trace_clock.h 4generic-y += trace_clock.h
5generic-y += preempt.h
diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild
index 8ada3cf0c98d..7e0e7213a481 100644
--- a/arch/h8300/include/asm/Kbuild
+++ b/arch/h8300/include/asm/Kbuild
@@ -6,3 +6,4 @@ generic-y += mmu.h
6generic-y += module.h 6generic-y += module.h
7generic-y += trace_clock.h 7generic-y += trace_clock.h
8generic-y += xor.h 8generic-y += xor.h
9generic-y += preempt.h
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild
index 1da17caac23c..67c3450309b7 100644
--- a/arch/hexagon/include/asm/Kbuild
+++ b/arch/hexagon/include/asm/Kbuild
@@ -53,3 +53,4 @@ generic-y += types.h
53generic-y += ucontext.h 53generic-y += ucontext.h
54generic-y += unaligned.h 54generic-y += unaligned.h
55generic-y += xor.h 55generic-y += xor.h
56generic-y += preempt.h
diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild
index a3456f34f672..f93ee087e8fe 100644
--- a/arch/ia64/include/asm/Kbuild
+++ b/arch/ia64/include/asm/Kbuild
@@ -3,4 +3,5 @@ generic-y += clkdev.h
3generic-y += exec.h 3generic-y += exec.h
4generic-y += kvm_para.h 4generic-y += kvm_para.h
5generic-y += trace_clock.h 5generic-y += trace_clock.h
6generic-y += preempt.h
6generic-y += vtime.h \ No newline at end of file 7generic-y += vtime.h \ No newline at end of file
diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild
index bebdc36ebb0a..2b58c5f0bc38 100644
--- a/arch/m32r/include/asm/Kbuild
+++ b/arch/m32r/include/asm/Kbuild
@@ -3,3 +3,4 @@ generic-y += clkdev.h
3generic-y += exec.h 3generic-y += exec.h
4generic-y += module.h 4generic-y += module.h
5generic-y += trace_clock.h 5generic-y += trace_clock.h
6generic-y += preempt.h
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild
index 09d77a862da3..a5d27f272a59 100644
--- a/arch/m68k/include/asm/Kbuild
+++ b/arch/m68k/include/asm/Kbuild
@@ -31,3 +31,4 @@ generic-y += trace_clock.h
31generic-y += types.h 31generic-y += types.h
32generic-y += word-at-a-time.h 32generic-y += word-at-a-time.h
33generic-y += xor.h 33generic-y += xor.h
34generic-y += preempt.h
diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild
index 6ae0ccb632cb..84d0c1d6b9b3 100644
--- a/arch/metag/include/asm/Kbuild
+++ b/arch/metag/include/asm/Kbuild
@@ -52,3 +52,4 @@ generic-y += unaligned.h
52generic-y += user.h 52generic-y += user.h
53generic-y += vga.h 53generic-y += vga.h
54generic-y += xor.h 54generic-y += xor.h
55generic-y += preempt.h
diff --git a/arch/metag/include/asm/topology.h b/arch/metag/include/asm/topology.h
index 23f5118f58db..8e9c0b3b9691 100644
--- a/arch/metag/include/asm/topology.h
+++ b/arch/metag/include/asm/topology.h
@@ -26,6 +26,8 @@
26 .last_balance = jiffies, \ 26 .last_balance = jiffies, \
27 .balance_interval = 1, \ 27 .balance_interval = 1, \
28 .nr_balance_failed = 0, \ 28 .nr_balance_failed = 0, \
29 .max_newidle_lb_cost = 0, \
30 .next_decay_max_lb_cost = jiffies, \
29} 31}
30 32
31#define cpu_to_node(cpu) ((void)(cpu), 0) 33#define cpu_to_node(cpu) ((void)(cpu), 0)
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index d3c51a6a601d..ce0bbf8f5640 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -3,3 +3,4 @@ generic-y += clkdev.h
3generic-y += exec.h 3generic-y += exec.h
4generic-y += trace_clock.h 4generic-y += trace_clock.h
5generic-y += syscalls.h 5generic-y += syscalls.h
6generic-y += preempt.h
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild
index 454ddf9bb76f..1acbb8b77a71 100644
--- a/arch/mips/include/asm/Kbuild
+++ b/arch/mips/include/asm/Kbuild
@@ -11,5 +11,6 @@ generic-y += sections.h
11generic-y += segment.h 11generic-y += segment.h
12generic-y += serial.h 12generic-y += serial.h
13generic-y += trace_clock.h 13generic-y += trace_clock.h
14generic-y += preempt.h
14generic-y += ucontext.h 15generic-y += ucontext.h
15generic-y += xor.h 16generic-y += xor.h
diff --git a/arch/mips/kernel/rtlx.c b/arch/mips/kernel/rtlx.c
index d763f11e35e2..2c12ea1668d1 100644
--- a/arch/mips/kernel/rtlx.c
+++ b/arch/mips/kernel/rtlx.c
@@ -172,8 +172,9 @@ int rtlx_open(int index, int can_sleep)
172 if (rtlx == NULL) { 172 if (rtlx == NULL) {
173 if( (p = vpe_get_shared(tclimit)) == NULL) { 173 if( (p = vpe_get_shared(tclimit)) == NULL) {
174 if (can_sleep) { 174 if (can_sleep) {
175 __wait_event_interruptible(channel_wqs[index].lx_queue, 175 ret = __wait_event_interruptible(
176 (p = vpe_get_shared(tclimit)), ret); 176 channel_wqs[index].lx_queue,
177 (p = vpe_get_shared(tclimit)));
177 if (ret) 178 if (ret)
178 goto out_fail; 179 goto out_fail;
179 } else { 180 } else {
@@ -263,11 +264,10 @@ unsigned int rtlx_read_poll(int index, int can_sleep)
263 /* data available to read? */ 264 /* data available to read? */
264 if (chan->lx_read == chan->lx_write) { 265 if (chan->lx_read == chan->lx_write) {
265 if (can_sleep) { 266 if (can_sleep) {
266 int ret = 0; 267 int ret = __wait_event_interruptible(
267 268 channel_wqs[index].lx_queue,
268 __wait_event_interruptible(channel_wqs[index].lx_queue,
269 (chan->lx_read != chan->lx_write) || 269 (chan->lx_read != chan->lx_write) ||
270 sp_stopping, ret); 270 sp_stopping);
271 if (ret) 271 if (ret)
272 return ret; 272 return ret;
273 273
@@ -440,14 +440,13 @@ static ssize_t file_write(struct file *file, const char __user * buffer,
440 440
441 /* any space left... */ 441 /* any space left... */
442 if (!rtlx_write_poll(minor)) { 442 if (!rtlx_write_poll(minor)) {
443 int ret = 0; 443 int ret;
444 444
445 if (file->f_flags & O_NONBLOCK) 445 if (file->f_flags & O_NONBLOCK)
446 return -EAGAIN; 446 return -EAGAIN;
447 447
448 __wait_event_interruptible(channel_wqs[minor].rt_queue, 448 ret = __wait_event_interruptible(channel_wqs[minor].rt_queue,
449 rtlx_write_poll(minor), 449 rtlx_write_poll(minor));
450 ret);
451 if (ret) 450 if (ret)
452 return ret; 451 return ret;
453 } 452 }
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index e205ef598e97..12156176c7ca 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -124,7 +124,7 @@ void *kmap_coherent(struct page *page, unsigned long addr)
124 124
125 BUG_ON(Page_dcache_dirty(page)); 125 BUG_ON(Page_dcache_dirty(page));
126 126
127 inc_preempt_count(); 127 pagefault_disable();
128 idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1); 128 idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
129#ifdef CONFIG_MIPS_MT_SMTC 129#ifdef CONFIG_MIPS_MT_SMTC
130 idx += FIX_N_COLOURS * smp_processor_id() + 130 idx += FIX_N_COLOURS * smp_processor_id() +
@@ -193,8 +193,7 @@ void kunmap_coherent(void)
193 write_c0_entryhi(old_ctx); 193 write_c0_entryhi(old_ctx);
194 EXIT_CRITICAL(flags); 194 EXIT_CRITICAL(flags);
195#endif 195#endif
196 dec_preempt_count(); 196 pagefault_enable();
197 preempt_check_resched();
198} 197}
199 198
200void copy_user_highpage(struct page *to, struct page *from, 199void copy_user_highpage(struct page *to, struct page *from,
diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild
index c5d767028306..74742dc6a3da 100644
--- a/arch/mn10300/include/asm/Kbuild
+++ b/arch/mn10300/include/asm/Kbuild
@@ -2,3 +2,4 @@
2generic-y += clkdev.h 2generic-y += clkdev.h
3generic-y += exec.h 3generic-y += exec.h
4generic-y += trace_clock.h 4generic-y += trace_clock.h
5generic-y += preempt.h
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index 195653e851da..78405625e799 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -67,3 +67,4 @@ generic-y += ucontext.h
67generic-y += user.h 67generic-y += user.h
68generic-y += word-at-a-time.h 68generic-y += word-at-a-time.h
69generic-y += xor.h 69generic-y += xor.h
70generic-y += preempt.h
diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild
index ff4c9faed546..a603b9ebe54c 100644
--- a/arch/parisc/include/asm/Kbuild
+++ b/arch/parisc/include/asm/Kbuild
@@ -4,3 +4,4 @@ generic-y += word-at-a-time.h auxvec.h user.h cputime.h emergency-restart.h \
4 div64.h irq_regs.h kdebug.h kvm_para.h local64.h local.h param.h \ 4 div64.h irq_regs.h kdebug.h kvm_para.h local64.h local.h param.h \
5 poll.h xor.h clkdev.h exec.h 5 poll.h xor.h clkdev.h exec.h
6generic-y += trace_clock.h 6generic-y += trace_clock.h
7generic-y += preempt.h
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild
index 704e6f10ae80..d8f9d2f18a23 100644
--- a/arch/powerpc/include/asm/Kbuild
+++ b/arch/powerpc/include/asm/Kbuild
@@ -2,4 +2,5 @@
2generic-y += clkdev.h 2generic-y += clkdev.h
3generic-y += rwsem.h 3generic-y += rwsem.h
4generic-y += trace_clock.h 4generic-y += trace_clock.h
5generic-y += preempt.h
5generic-y += vtime.h \ No newline at end of file 6generic-y += vtime.h \ No newline at end of file
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild
index f313f9cbcf44..7a5288f3479a 100644
--- a/arch/s390/include/asm/Kbuild
+++ b/arch/s390/include/asm/Kbuild
@@ -2,3 +2,4 @@
2 2
3generic-y += clkdev.h 3generic-y += clkdev.h
4generic-y += trace_clock.h 4generic-y += trace_clock.h
5generic-y += preempt.h
diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild
index e1c7bb999b06..f3414ade77a3 100644
--- a/arch/score/include/asm/Kbuild
+++ b/arch/score/include/asm/Kbuild
@@ -4,3 +4,4 @@ header-y +=
4generic-y += clkdev.h 4generic-y += clkdev.h
5generic-y += trace_clock.h 5generic-y += trace_clock.h
6generic-y += xor.h 6generic-y += xor.h
7generic-y += preempt.h
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild
index 280bea9e5e2b..231efbb68108 100644
--- a/arch/sh/include/asm/Kbuild
+++ b/arch/sh/include/asm/Kbuild
@@ -34,3 +34,4 @@ generic-y += termios.h
34generic-y += trace_clock.h 34generic-y += trace_clock.h
35generic-y += ucontext.h 35generic-y += ucontext.h
36generic-y += xor.h 36generic-y += xor.h
37generic-y += preempt.h
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index 7e4a97fbded4..bf390667657a 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -16,3 +16,4 @@ generic-y += serial.h
16generic-y += trace_clock.h 16generic-y += trace_clock.h
17generic-y += types.h 17generic-y += types.h
18generic-y += word-at-a-time.h 18generic-y += word-at-a-time.h
19generic-y += preempt.h
diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild
index 664d6ad23f80..22f3bd147fa7 100644
--- a/arch/tile/include/asm/Kbuild
+++ b/arch/tile/include/asm/Kbuild
@@ -38,3 +38,4 @@ generic-y += termios.h
38generic-y += trace_clock.h 38generic-y += trace_clock.h
39generic-y += types.h 39generic-y += types.h
40generic-y += xor.h 40generic-y += xor.h
41generic-y += preempt.h
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild
index b30f34a79882..fdde187e6087 100644
--- a/arch/um/include/asm/Kbuild
+++ b/arch/um/include/asm/Kbuild
@@ -3,3 +3,4 @@ generic-y += hw_irq.h irq_regs.h kdebug.h percpu.h sections.h topology.h xor.h
3generic-y += ftrace.h pci.h io.h param.h delay.h mutex.h current.h exec.h 3generic-y += ftrace.h pci.h io.h param.h delay.h mutex.h current.h exec.h
4generic-y += switch_to.h clkdev.h 4generic-y += switch_to.h clkdev.h
5generic-y += trace_clock.h 5generic-y += trace_clock.h
6generic-y += preempt.h
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild
index 89d8b6c4e39a..00045cbe5c63 100644
--- a/arch/unicore32/include/asm/Kbuild
+++ b/arch/unicore32/include/asm/Kbuild
@@ -60,3 +60,4 @@ generic-y += unaligned.h
60generic-y += user.h 60generic-y += user.h
61generic-y += vga.h 61generic-y += vga.h
62generic-y += xor.h 62generic-y += xor.h
63generic-y += preempt.h
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 722aa3b04624..da31c8b8a92d 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -6,6 +6,7 @@
6#include <asm/processor.h> 6#include <asm/processor.h>
7#include <asm/alternative.h> 7#include <asm/alternative.h>
8#include <asm/cmpxchg.h> 8#include <asm/cmpxchg.h>
9#include <asm/rmwcc.h>
9 10
10/* 11/*
11 * Atomic operations that C can't guarantee us. Useful for 12 * Atomic operations that C can't guarantee us. Useful for
@@ -76,12 +77,7 @@ static inline void atomic_sub(int i, atomic_t *v)
76 */ 77 */
77static inline int atomic_sub_and_test(int i, atomic_t *v) 78static inline int atomic_sub_and_test(int i, atomic_t *v)
78{ 79{
79 unsigned char c; 80 GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, i, "%0", "e");
80
81 asm volatile(LOCK_PREFIX "subl %2,%0; sete %1"
82 : "+m" (v->counter), "=qm" (c)
83 : "ir" (i) : "memory");
84 return c;
85} 81}
86 82
87/** 83/**
@@ -118,12 +114,7 @@ static inline void atomic_dec(atomic_t *v)
118 */ 114 */
119static inline int atomic_dec_and_test(atomic_t *v) 115static inline int atomic_dec_and_test(atomic_t *v)
120{ 116{
121 unsigned char c; 117 GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", "e");
122
123 asm volatile(LOCK_PREFIX "decl %0; sete %1"
124 : "+m" (v->counter), "=qm" (c)
125 : : "memory");
126 return c != 0;
127} 118}
128 119
129/** 120/**
@@ -136,12 +127,7 @@ static inline int atomic_dec_and_test(atomic_t *v)
136 */ 127 */
137static inline int atomic_inc_and_test(atomic_t *v) 128static inline int atomic_inc_and_test(atomic_t *v)
138{ 129{
139 unsigned char c; 130 GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", "e");
140
141 asm volatile(LOCK_PREFIX "incl %0; sete %1"
142 : "+m" (v->counter), "=qm" (c)
143 : : "memory");
144 return c != 0;
145} 131}
146 132
147/** 133/**
@@ -155,12 +141,7 @@ static inline int atomic_inc_and_test(atomic_t *v)
155 */ 141 */
156static inline int atomic_add_negative(int i, atomic_t *v) 142static inline int atomic_add_negative(int i, atomic_t *v)
157{ 143{
158 unsigned char c; 144 GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, i, "%0", "s");
159
160 asm volatile(LOCK_PREFIX "addl %2,%0; sets %1"
161 : "+m" (v->counter), "=qm" (c)
162 : "ir" (i) : "memory");
163 return c;
164} 145}
165 146
166/** 147/**
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 0e1cbfc8ee06..3f065c985aee 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -72,12 +72,7 @@ static inline void atomic64_sub(long i, atomic64_t *v)
72 */ 72 */
73static inline int atomic64_sub_and_test(long i, atomic64_t *v) 73static inline int atomic64_sub_and_test(long i, atomic64_t *v)
74{ 74{
75 unsigned char c; 75 GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, i, "%0", "e");
76
77 asm volatile(LOCK_PREFIX "subq %2,%0; sete %1"
78 : "=m" (v->counter), "=qm" (c)
79 : "er" (i), "m" (v->counter) : "memory");
80 return c;
81} 76}
82 77
83/** 78/**
@@ -116,12 +111,7 @@ static inline void atomic64_dec(atomic64_t *v)
116 */ 111 */
117static inline int atomic64_dec_and_test(atomic64_t *v) 112static inline int atomic64_dec_and_test(atomic64_t *v)
118{ 113{
119 unsigned char c; 114 GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", "e");
120
121 asm volatile(LOCK_PREFIX "decq %0; sete %1"
122 : "=m" (v->counter), "=qm" (c)
123 : "m" (v->counter) : "memory");
124 return c != 0;
125} 115}
126 116
127/** 117/**
@@ -134,12 +124,7 @@ static inline int atomic64_dec_and_test(atomic64_t *v)
134 */ 124 */
135static inline int atomic64_inc_and_test(atomic64_t *v) 125static inline int atomic64_inc_and_test(atomic64_t *v)
136{ 126{
137 unsigned char c; 127 GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", "e");
138
139 asm volatile(LOCK_PREFIX "incq %0; sete %1"
140 : "=m" (v->counter), "=qm" (c)
141 : "m" (v->counter) : "memory");
142 return c != 0;
143} 128}
144 129
145/** 130/**
@@ -153,12 +138,7 @@ static inline int atomic64_inc_and_test(atomic64_t *v)
153 */ 138 */
154static inline int atomic64_add_negative(long i, atomic64_t *v) 139static inline int atomic64_add_negative(long i, atomic64_t *v)
155{ 140{
156 unsigned char c; 141 GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, i, "%0", "s");
157
158 asm volatile(LOCK_PREFIX "addq %2,%0; sets %1"
159 : "=m" (v->counter), "=qm" (c)
160 : "er" (i), "m" (v->counter) : "memory");
161 return c;
162} 142}
163 143
164/** 144/**
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 41639ce8fd63..6d76d0935989 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -14,6 +14,7 @@
14 14
15#include <linux/compiler.h> 15#include <linux/compiler.h>
16#include <asm/alternative.h> 16#include <asm/alternative.h>
17#include <asm/rmwcc.h>
17 18
18#if BITS_PER_LONG == 32 19#if BITS_PER_LONG == 32
19# define _BITOPS_LONG_SHIFT 5 20# define _BITOPS_LONG_SHIFT 5
@@ -204,12 +205,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
204 */ 205 */
205static inline int test_and_set_bit(long nr, volatile unsigned long *addr) 206static inline int test_and_set_bit(long nr, volatile unsigned long *addr)
206{ 207{
207 int oldbit; 208 GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, nr, "%0", "c");
208
209 asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
210 "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
211
212 return oldbit;
213} 209}
214 210
215/** 211/**
@@ -255,13 +251,7 @@ static inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
255 */ 251 */
256static inline int test_and_clear_bit(long nr, volatile unsigned long *addr) 252static inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
257{ 253{
258 int oldbit; 254 GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, nr, "%0", "c");
259
260 asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
261 "sbb %0,%0"
262 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
263
264 return oldbit;
265} 255}
266 256
267/** 257/**
@@ -314,13 +304,7 @@ static inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
314 */ 304 */
315static inline int test_and_change_bit(long nr, volatile unsigned long *addr) 305static inline int test_and_change_bit(long nr, volatile unsigned long *addr)
316{ 306{
317 int oldbit; 307 GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, nr, "%0", "c");
318
319 asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
320 "sbb %0,%0"
321 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
322
323 return oldbit;
324} 308}
325 309
326static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr) 310static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr)
diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h
index 0fa675033912..cb4c73bfeb48 100644
--- a/arch/x86/include/asm/calling.h
+++ b/arch/x86/include/asm/calling.h
@@ -48,6 +48,8 @@ For 32-bit we have the following conventions - kernel is built with
48 48
49#include <asm/dwarf2.h> 49#include <asm/dwarf2.h>
50 50
51#ifdef CONFIG_X86_64
52
51/* 53/*
52 * 64-bit system call stack frame layout defines and helpers, 54 * 64-bit system call stack frame layout defines and helpers,
53 * for assembly code: 55 * for assembly code:
@@ -192,3 +194,51 @@ For 32-bit we have the following conventions - kernel is built with
192 .macro icebp 194 .macro icebp
193 .byte 0xf1 195 .byte 0xf1
194 .endm 196 .endm
197
198#else /* CONFIG_X86_64 */
199
200/*
201 * For 32bit only simplified versions of SAVE_ALL/RESTORE_ALL. These
202 * are different from the entry_32.S versions in not changing the segment
203 * registers. So only suitable for in kernel use, not when transitioning
204 * from or to user space. The resulting stack frame is not a standard
205 * pt_regs frame. The main use case is calling C code from assembler
206 * when all the registers need to be preserved.
207 */
208
209 .macro SAVE_ALL
210 pushl_cfi %eax
211 CFI_REL_OFFSET eax, 0
212 pushl_cfi %ebp
213 CFI_REL_OFFSET ebp, 0
214 pushl_cfi %edi
215 CFI_REL_OFFSET edi, 0
216 pushl_cfi %esi
217 CFI_REL_OFFSET esi, 0
218 pushl_cfi %edx
219 CFI_REL_OFFSET edx, 0
220 pushl_cfi %ecx
221 CFI_REL_OFFSET ecx, 0
222 pushl_cfi %ebx
223 CFI_REL_OFFSET ebx, 0
224 .endm
225
226 .macro RESTORE_ALL
227 popl_cfi %ebx
228 CFI_RESTORE ebx
229 popl_cfi %ecx
230 CFI_RESTORE ecx
231 popl_cfi %edx
232 CFI_RESTORE edx
233 popl_cfi %esi
234 CFI_RESTORE esi
235 popl_cfi %edi
236 CFI_RESTORE edi
237 popl_cfi %ebp
238 CFI_RESTORE ebp
239 popl_cfi %eax
240 CFI_RESTORE eax
241 .endm
242
243#endif /* CONFIG_X86_64 */
244
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 2d89e3980cbd..5b23e605e707 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -52,12 +52,7 @@ static inline void local_sub(long i, local_t *l)
52 */ 52 */
53static inline int local_sub_and_test(long i, local_t *l) 53static inline int local_sub_and_test(long i, local_t *l)
54{ 54{
55 unsigned char c; 55 GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, i, "%0", "e");
56
57 asm volatile(_ASM_SUB "%2,%0; sete %1"
58 : "+m" (l->a.counter), "=qm" (c)
59 : "ir" (i) : "memory");
60 return c;
61} 56}
62 57
63/** 58/**
@@ -70,12 +65,7 @@ static inline int local_sub_and_test(long i, local_t *l)
70 */ 65 */
71static inline int local_dec_and_test(local_t *l) 66static inline int local_dec_and_test(local_t *l)
72{ 67{
73 unsigned char c; 68 GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", "e");
74
75 asm volatile(_ASM_DEC "%0; sete %1"
76 : "+m" (l->a.counter), "=qm" (c)
77 : : "memory");
78 return c != 0;
79} 69}
80 70
81/** 71/**
@@ -88,12 +78,7 @@ static inline int local_dec_and_test(local_t *l)
88 */ 78 */
89static inline int local_inc_and_test(local_t *l) 79static inline int local_inc_and_test(local_t *l)
90{ 80{
91 unsigned char c; 81 GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", "e");
92
93 asm volatile(_ASM_INC "%0; sete %1"
94 : "+m" (l->a.counter), "=qm" (c)
95 : : "memory");
96 return c != 0;
97} 82}
98 83
99/** 84/**
@@ -107,12 +92,7 @@ static inline int local_inc_and_test(local_t *l)
107 */ 92 */
108static inline int local_add_negative(long i, local_t *l) 93static inline int local_add_negative(long i, local_t *l)
109{ 94{
110 unsigned char c; 95 GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, i, "%0", "s");
111
112 asm volatile(_ASM_ADD "%2,%0; sets %1"
113 : "+m" (l->a.counter), "=qm" (c)
114 : "ir" (i) : "memory");
115 return c;
116} 96}
117 97
118/** 98/**
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
new file mode 100644
index 000000000000..8729723636fd
--- /dev/null
+++ b/arch/x86/include/asm/preempt.h
@@ -0,0 +1,100 @@
1#ifndef __ASM_PREEMPT_H
2#define __ASM_PREEMPT_H
3
4#include <asm/rmwcc.h>
5#include <asm/percpu.h>
6#include <linux/thread_info.h>
7
8DECLARE_PER_CPU(int, __preempt_count);
9
10/*
11 * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
12 * that think a non-zero value indicates we cannot preempt.
13 */
14static __always_inline int preempt_count(void)
15{
16 return __this_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
17}
18
19static __always_inline void preempt_count_set(int pc)
20{
21 __this_cpu_write_4(__preempt_count, pc);
22}
23
24/*
25 * must be macros to avoid header recursion hell
26 */
27#define task_preempt_count(p) \
28 (task_thread_info(p)->saved_preempt_count & ~PREEMPT_NEED_RESCHED)
29
30#define init_task_preempt_count(p) do { \
31 task_thread_info(p)->saved_preempt_count = PREEMPT_DISABLED; \
32} while (0)
33
34#define init_idle_preempt_count(p, cpu) do { \
35 task_thread_info(p)->saved_preempt_count = PREEMPT_ENABLED; \
36 per_cpu(__preempt_count, (cpu)) = PREEMPT_ENABLED; \
37} while (0)
38
39/*
40 * We fold the NEED_RESCHED bit into the preempt count such that
41 * preempt_enable() can decrement and test for needing to reschedule with a
42 * single instruction.
43 *
44 * We invert the actual bit, so that when the decrement hits 0 we know we both
45 * need to resched (the bit is cleared) and can resched (no preempt count).
46 */
47
48static __always_inline void set_preempt_need_resched(void)
49{
50 __this_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
51}
52
53static __always_inline void clear_preempt_need_resched(void)
54{
55 __this_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
56}
57
58static __always_inline bool test_preempt_need_resched(void)
59{
60 return !(__this_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
61}
62
63/*
64 * The various preempt_count add/sub methods
65 */
66
67static __always_inline void __preempt_count_add(int val)
68{
69 __this_cpu_add_4(__preempt_count, val);
70}
71
72static __always_inline void __preempt_count_sub(int val)
73{
74 __this_cpu_add_4(__preempt_count, -val);
75}
76
77static __always_inline bool __preempt_count_dec_and_test(void)
78{
79 GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e");
80}
81
82/*
83 * Returns true when we need to resched and can (barring IRQ state).
84 */
85static __always_inline bool should_resched(void)
86{
87 return unlikely(!__this_cpu_read_4(__preempt_count));
88}
89
90#ifdef CONFIG_PREEMPT
91 extern asmlinkage void ___preempt_schedule(void);
92# define __preempt_schedule() asm ("call ___preempt_schedule")
93 extern asmlinkage void preempt_schedule(void);
94# ifdef CONFIG_CONTEXT_TRACKING
95 extern asmlinkage void ___preempt_schedule_context(void);
96# define __preempt_schedule_context() asm ("call ___preempt_schedule_context")
97# endif
98#endif
99
100#endif /* __ASM_PREEMPT_H */
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
new file mode 100644
index 000000000000..1ff990f1de8e
--- /dev/null
+++ b/arch/x86/include/asm/rmwcc.h
@@ -0,0 +1,41 @@
1#ifndef _ASM_X86_RMWcc
2#define _ASM_X86_RMWcc
3
4#ifdef CC_HAVE_ASM_GOTO
5
6#define __GEN_RMWcc(fullop, var, cc, ...) \
7do { \
8 asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \
9 : : "m" (var), ## __VA_ARGS__ \
10 : "memory" : cc_label); \
11 return 0; \
12cc_label: \
13 return 1; \
14} while (0)
15
16#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
17 __GEN_RMWcc(op " " arg0, var, cc)
18
19#define GEN_BINARY_RMWcc(op, var, val, arg0, cc) \
20 __GEN_RMWcc(op " %1, " arg0, var, cc, "er" (val))
21
22#else /* !CC_HAVE_ASM_GOTO */
23
24#define __GEN_RMWcc(fullop, var, cc, ...) \
25do { \
26 char c; \
27 asm volatile (fullop "; set" cc " %1" \
28 : "+m" (var), "=qm" (c) \
29 : __VA_ARGS__ : "memory"); \
30 return c != 0; \
31} while (0)
32
33#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
34 __GEN_RMWcc(op " " arg0, var, cc)
35
36#define GEN_BINARY_RMWcc(op, var, val, arg0, cc) \
37 __GEN_RMWcc(op " %2, " arg0, var, cc, "er" (val))
38
39#endif /* CC_HAVE_ASM_GOTO */
40
41#endif /* _ASM_X86_RMWcc */
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 27811190cbd7..c46a46be1ec6 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -28,8 +28,7 @@ struct thread_info {
28 __u32 flags; /* low level flags */ 28 __u32 flags; /* low level flags */
29 __u32 status; /* thread synchronous flags */ 29 __u32 status; /* thread synchronous flags */
30 __u32 cpu; /* current CPU */ 30 __u32 cpu; /* current CPU */
31 int preempt_count; /* 0 => preemptable, 31 int saved_preempt_count;
32 <0 => BUG */
33 mm_segment_t addr_limit; 32 mm_segment_t addr_limit;
34 struct restart_block restart_block; 33 struct restart_block restart_block;
35 void __user *sysenter_return; 34 void __user *sysenter_return;
@@ -49,7 +48,7 @@ struct thread_info {
49 .exec_domain = &default_exec_domain, \ 48 .exec_domain = &default_exec_domain, \
50 .flags = 0, \ 49 .flags = 0, \
51 .cpu = 0, \ 50 .cpu = 0, \
52 .preempt_count = INIT_PREEMPT_COUNT, \ 51 .saved_preempt_count = INIT_PREEMPT_COUNT, \
53 .addr_limit = KERNEL_DS, \ 52 .addr_limit = KERNEL_DS, \
54 .restart_block = { \ 53 .restart_block = { \
55 .fn = do_no_restart_syscall, \ 54 .fn = do_no_restart_syscall, \
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index a5408b965c9d..9b0a34e2cd79 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -36,6 +36,8 @@ obj-y += tsc.o io_delay.o rtc.o
36obj-y += pci-iommu_table.o 36obj-y += pci-iommu_table.o
37obj-y += resource.o 37obj-y += resource.o
38 38
39obj-$(CONFIG_PREEMPT) += preempt.o
40
39obj-y += process.o 41obj-y += process.o
40obj-y += i387.o xsave.o 42obj-y += i387.o xsave.o
41obj-y += ptrace.o 43obj-y += ptrace.o
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 28610822fb3c..9f6b9341950f 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -32,7 +32,6 @@ void common(void) {
32 OFFSET(TI_flags, thread_info, flags); 32 OFFSET(TI_flags, thread_info, flags);
33 OFFSET(TI_status, thread_info, status); 33 OFFSET(TI_status, thread_info, status);
34 OFFSET(TI_addr_limit, thread_info, addr_limit); 34 OFFSET(TI_addr_limit, thread_info, addr_limit);
35 OFFSET(TI_preempt_count, thread_info, preempt_count);
36 35
37 BLANK(); 36 BLANK();
38 OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx); 37 OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 2793d1f095a2..5223fe6dec7b 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1095,6 +1095,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
1095 1095
1096DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1; 1096DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
1097 1097
1098DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
1099EXPORT_PER_CPU_SYMBOL(__preempt_count);
1100
1098DEFINE_PER_CPU(struct task_struct *, fpu_owner_task); 1101DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
1099 1102
1100/* 1103/*
@@ -1169,6 +1172,8 @@ void debug_stack_reset(void)
1169 1172
1170DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; 1173DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
1171EXPORT_PER_CPU_SYMBOL(current_task); 1174EXPORT_PER_CPU_SYMBOL(current_task);
1175DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
1176EXPORT_PER_CPU_SYMBOL(__preempt_count);
1172DEFINE_PER_CPU(struct task_struct *, fpu_owner_task); 1177DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
1173 1178
1174#ifdef CONFIG_CC_STACKPROTECTOR 1179#ifdef CONFIG_CC_STACKPROTECTOR
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index f0dcb0ceb6a2..fd1bc1b15e6d 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -362,12 +362,9 @@ END(ret_from_exception)
362#ifdef CONFIG_PREEMPT 362#ifdef CONFIG_PREEMPT
363ENTRY(resume_kernel) 363ENTRY(resume_kernel)
364 DISABLE_INTERRUPTS(CLBR_ANY) 364 DISABLE_INTERRUPTS(CLBR_ANY)
365 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
366 jnz restore_all
367need_resched: 365need_resched:
368 movl TI_flags(%ebp), %ecx # need_resched set ? 366 cmpl $0,PER_CPU_VAR(__preempt_count)
369 testb $_TIF_NEED_RESCHED, %cl 367 jnz restore_all
370 jz restore_all
371 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ? 368 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
372 jz restore_all 369 jz restore_all
373 call preempt_schedule_irq 370 call preempt_schedule_irq
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 083da7c2f40d..603be7c70675 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1103,10 +1103,8 @@ retint_signal:
1103 /* Returning to kernel space. Check if we need preemption */ 1103 /* Returning to kernel space. Check if we need preemption */
1104 /* rcx: threadinfo. interrupts off. */ 1104 /* rcx: threadinfo. interrupts off. */
1105ENTRY(retint_kernel) 1105ENTRY(retint_kernel)
1106 cmpl $0,TI_preempt_count(%rcx) 1106 cmpl $0,PER_CPU_VAR(__preempt_count)
1107 jnz retint_restore_args 1107 jnz retint_restore_args
1108 bt $TIF_NEED_RESCHED,TI_flags(%rcx)
1109 jnc retint_restore_args
1110 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */ 1108 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
1111 jnc retint_restore_args 1109 jnc retint_restore_args
1112 call preempt_schedule_irq 1110 call preempt_schedule_irq
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index 0fa69127209a..05fd74f537d6 100644
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -37,3 +37,10 @@ EXPORT_SYMBOL(strstr);
37 37
38EXPORT_SYMBOL(csum_partial); 38EXPORT_SYMBOL(csum_partial);
39EXPORT_SYMBOL(empty_zero_page); 39EXPORT_SYMBOL(empty_zero_page);
40
41#ifdef CONFIG_PREEMPT
42EXPORT_SYMBOL(___preempt_schedule);
43#ifdef CONFIG_CONTEXT_TRACKING
44EXPORT_SYMBOL(___preempt_schedule_context);
45#endif
46#endif
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 8a5bb01dbc0e..d7fcbedc9c43 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -100,9 +100,6 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
100 irqctx->tinfo.task = curctx->tinfo.task; 100 irqctx->tinfo.task = curctx->tinfo.task;
101 irqctx->tinfo.previous_esp = current_stack_pointer; 101 irqctx->tinfo.previous_esp = current_stack_pointer;
102 102
103 /* Copy the preempt_count so that the [soft]irq checks work. */
104 irqctx->tinfo.preempt_count = curctx->tinfo.preempt_count;
105
106 if (unlikely(overflow)) 103 if (unlikely(overflow))
107 call_on_stack(print_stack_overflow, isp); 104 call_on_stack(print_stack_overflow, isp);
108 105
@@ -131,7 +128,6 @@ void irq_ctx_init(int cpu)
131 THREAD_SIZE_ORDER)); 128 THREAD_SIZE_ORDER));
132 memset(&irqctx->tinfo, 0, sizeof(struct thread_info)); 129 memset(&irqctx->tinfo, 0, sizeof(struct thread_info));
133 irqctx->tinfo.cpu = cpu; 130 irqctx->tinfo.cpu = cpu;
134 irqctx->tinfo.preempt_count = HARDIRQ_OFFSET;
135 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); 131 irqctx->tinfo.addr_limit = MAKE_MM_SEG(0);
136 132
137 per_cpu(hardirq_ctx, cpu) = irqctx; 133 per_cpu(hardirq_ctx, cpu) = irqctx;
diff --git a/arch/x86/kernel/preempt.S b/arch/x86/kernel/preempt.S
new file mode 100644
index 000000000000..ca7f0d58a87d
--- /dev/null
+++ b/arch/x86/kernel/preempt.S
@@ -0,0 +1,25 @@
1
2#include <linux/linkage.h>
3#include <asm/dwarf2.h>
4#include <asm/asm.h>
5#include <asm/calling.h>
6
7ENTRY(___preempt_schedule)
8 CFI_STARTPROC
9 SAVE_ALL
10 call preempt_schedule
11 RESTORE_ALL
12 ret
13 CFI_ENDPROC
14
15#ifdef CONFIG_CONTEXT_TRACKING
16
17ENTRY(___preempt_schedule_context)
18 CFI_STARTPROC
19 SAVE_ALL
20 call preempt_schedule_context
21 RESTORE_ALL
22 ret
23 CFI_ENDPROC
24
25#endif
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index c83516be1052..3fb8d95ab8b5 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -391,9 +391,9 @@ static void amd_e400_idle(void)
391 * The switch back from broadcast mode needs to be 391 * The switch back from broadcast mode needs to be
392 * called with interrupts disabled. 392 * called with interrupts disabled.
393 */ 393 */
394 local_irq_disable(); 394 local_irq_disable();
395 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu); 395 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
396 local_irq_enable(); 396 local_irq_enable();
397 } else 397 } else
398 default_idle(); 398 default_idle();
399} 399}
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 884f98f69354..c2ec1aa6d454 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -292,6 +292,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
292 set_iopl_mask(next->iopl); 292 set_iopl_mask(next->iopl);
293 293
294 /* 294 /*
295 * If it were not for PREEMPT_ACTIVE we could guarantee that the
296 * preempt_count of all tasks was equal here and this would not be
297 * needed.
298 */
299 task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
300 this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
301
302 /*
295 * Now maybe handle debug registers and/or IO bitmaps 303 * Now maybe handle debug registers and/or IO bitmaps
296 */ 304 */
297 if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV || 305 if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV ||
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index bb1dc51bab05..45ab4d6fc8a7 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -363,6 +363,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
363 this_cpu_write(old_rsp, next->usersp); 363 this_cpu_write(old_rsp, next->usersp);
364 this_cpu_write(current_task, next_p); 364 this_cpu_write(current_task, next_p);
365 365
366 /*
367 * If it were not for PREEMPT_ACTIVE we could guarantee that the
368 * preempt_count of all tasks was equal here and this would not be
369 * needed.
370 */
371 task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
372 this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
373
366 this_cpu_write(kernel_stack, 374 this_cpu_write(kernel_stack,
367 (unsigned long)task_stack_page(next_p) + 375 (unsigned long)task_stack_page(next_p) +
368 THREAD_SIZE - KERNEL_STACK_OFFSET); 376 THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 8c8093b146ca..729aa779ff75 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -88,7 +88,7 @@ static inline void conditional_sti(struct pt_regs *regs)
88 88
89static inline void preempt_conditional_sti(struct pt_regs *regs) 89static inline void preempt_conditional_sti(struct pt_regs *regs)
90{ 90{
91 inc_preempt_count(); 91 preempt_count_inc();
92 if (regs->flags & X86_EFLAGS_IF) 92 if (regs->flags & X86_EFLAGS_IF)
93 local_irq_enable(); 93 local_irq_enable();
94} 94}
@@ -103,7 +103,7 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
103{ 103{
104 if (regs->flags & X86_EFLAGS_IF) 104 if (regs->flags & X86_EFLAGS_IF)
105 local_irq_disable(); 105 local_irq_disable();
106 dec_preempt_count(); 106 preempt_count_dec();
107} 107}
108 108
109static int __kprobes 109static int __kprobes
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index b014d9414d08..040681928e9d 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -66,3 +66,10 @@ EXPORT_SYMBOL(empty_zero_page);
66#ifndef CONFIG_PARAVIRT 66#ifndef CONFIG_PARAVIRT
67EXPORT_SYMBOL(native_load_gs_index); 67EXPORT_SYMBOL(native_load_gs_index);
68#endif 68#endif
69
70#ifdef CONFIG_PREEMPT
71EXPORT_SYMBOL(___preempt_schedule);
72#ifdef CONFIG_CONTEXT_TRACKING
73EXPORT_SYMBOL(___preempt_schedule_context);
74#endif
75#endif
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index 1b982641ec35..228d6aee3a16 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -28,3 +28,4 @@ generic-y += termios.h
28generic-y += topology.h 28generic-y += topology.h
29generic-y += trace_clock.h 29generic-y += trace_clock.h
30generic-y += xor.h 30generic-y += xor.h
31generic-y += preempt.h
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-14 11:15:42 -0400