diff options
30 files changed, 72 insertions, 57 deletions
diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 9d0ac091a52a..4a905bd667e2 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c | |||
@@ -23,8 +23,7 @@ | |||
23 | #include <linux/smp.h> | 23 | #include <linux/smp.h> |
24 | #include <linux/interrupt.h> | 24 | #include <linux/interrupt.h> |
25 | #include <linux/module.h> | 25 | #include <linux/module.h> |
26 | 26 | #include <linux/uaccess.h> | |
27 | #include <asm/uaccess.h> | ||
28 | 27 | ||
29 | extern void die_if_kernel(char *,struct pt_regs *,long, unsigned long *); | 28 | extern void die_if_kernel(char *,struct pt_regs *,long, unsigned long *); |
30 | 29 | ||
@@ -107,7 +106,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, | |||
107 | 106 | ||
108 | /* If we're in an interrupt context, or have no user context, | 107 | /* If we're in an interrupt context, or have no user context, |
109 | we must not take the fault. */ | 108 | we must not take the fault. */ |
110 | if (!mm || in_atomic()) | 109 | if (!mm || faulthandler_disabled()) |
111 | goto no_context; | 110 | goto no_context; |
112 | 111 | ||
113 | #ifdef CONFIG_ALPHA_LARGE_VMALLOC | 112 | #ifdef CONFIG_ALPHA_LARGE_VMALLOC |
diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 6a2e006cbcce..d948e4e9d89c 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c | |||
@@ -86,7 +86,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs) | |||
86 | * If we're in an interrupt or have no user | 86 | * If we're in an interrupt or have no user |
87 | * context, we must not take the fault.. | 87 | * context, we must not take the fault.. |
88 | */ | 88 | */ |
89 | if (in_atomic() || !mm) | 89 | if (faulthandler_disabled() || !mm) |
90 | goto no_context; | 90 | goto no_context; |
91 | 91 | ||
92 | if (user_mode(regs)) | 92 | if (user_mode(regs)) |
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 6333d9c17875..0d629b8f973f 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c | |||
@@ -276,7 +276,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) | |||
276 | * If we're in an interrupt or have no user | 276 | * If we're in an interrupt or have no user |
277 | * context, we must not take the fault.. | 277 | * context, we must not take the fault.. |
278 | */ | 278 | */ |
279 | if (in_atomic() || !mm) | 279 | if (faulthandler_disabled() || !mm) |
280 | goto no_context; | 280 | goto no_context; |
281 | 281 | ||
282 | if (user_mode(regs)) | 282 | if (user_mode(regs)) |
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 96da13167d4a..0948d327d013 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c | |||
@@ -211,7 +211,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, | |||
211 | * If we're in an interrupt or have no user context, we must not take | 211 | * If we're in an interrupt or have no user context, we must not take |
212 | * the fault. | 212 | * the fault. |
213 | */ | 213 | */ |
214 | if (in_atomic() || !mm) | 214 | if (faulthandler_disabled() || !mm) |
215 | goto no_context; | 215 | goto no_context; |
216 | 216 | ||
217 | if (user_mode(regs)) | 217 | if (user_mode(regs)) |
diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c index d223a8b57c1e..c03533937a9f 100644 --- a/arch/avr32/mm/fault.c +++ b/arch/avr32/mm/fault.c | |||
@@ -14,11 +14,11 @@ | |||
14 | #include <linux/pagemap.h> | 14 | #include <linux/pagemap.h> |
15 | #include <linux/kdebug.h> | 15 | #include <linux/kdebug.h> |
16 | #include <linux/kprobes.h> | 16 | #include <linux/kprobes.h> |
17 | #include <linux/uaccess.h> | ||
17 | 18 | ||
18 | #include <asm/mmu_context.h> | 19 | #include <asm/mmu_context.h> |
19 | #include <asm/sysreg.h> | 20 | #include <asm/sysreg.h> |
20 | #include <asm/tlb.h> | 21 | #include <asm/tlb.h> |
21 | #include <asm/uaccess.h> | ||
22 | 22 | ||
23 | #ifdef CONFIG_KPROBES | 23 | #ifdef CONFIG_KPROBES |
24 | static inline int notify_page_fault(struct pt_regs *regs, int trap) | 24 | static inline int notify_page_fault(struct pt_regs *regs, int trap) |
@@ -81,7 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs) | |||
81 | * If we're in an interrupt or have no user context, we must | 81 | * If we're in an interrupt or have no user context, we must |
82 | * not take the fault... | 82 | * not take the fault... |
83 | */ | 83 | */ |
84 | if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM)) | 84 | if (faulthandler_disabled() || !mm || regs->sr & SYSREG_BIT(GM)) |
85 | goto no_context; | 85 | goto no_context; |
86 | 86 | ||
87 | local_irq_enable(); | 87 | local_irq_enable(); |
diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c index 83f12f2ed9e3..3066d40a6db1 100644 --- a/arch/cris/mm/fault.c +++ b/arch/cris/mm/fault.c | |||
@@ -8,7 +8,7 @@ | |||
8 | #include <linux/interrupt.h> | 8 | #include <linux/interrupt.h> |
9 | #include <linux/module.h> | 9 | #include <linux/module.h> |
10 | #include <linux/wait.h> | 10 | #include <linux/wait.h> |
11 | #include <asm/uaccess.h> | 11 | #include <linux/uaccess.h> |
12 | #include <arch/system.h> | 12 | #include <arch/system.h> |
13 | 13 | ||
14 | extern int find_fixup_code(struct pt_regs *); | 14 | extern int find_fixup_code(struct pt_regs *); |
@@ -109,11 +109,11 @@ do_page_fault(unsigned long address, struct pt_regs *regs, | |||
109 | info.si_code = SEGV_MAPERR; | 109 | info.si_code = SEGV_MAPERR; |
110 | 110 | ||
111 | /* | 111 | /* |
112 | * If we're in an interrupt or "atomic" operation or have no | 112 | * If we're in an interrupt, have pagefaults disabled or have no |
113 | * user context, we must not take the fault. | 113 | * user context, we must not take the fault. |
114 | */ | 114 | */ |
115 | 115 | ||
116 | if (in_atomic() || !mm) | 116 | if (faulthandler_disabled() || !mm) |
117 | goto no_context; | 117 | goto no_context; |
118 | 118 | ||
119 | if (user_mode(regs)) | 119 | if (user_mode(regs)) |
diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c index ec4917ddf678..61d99767fe16 100644 --- a/arch/frv/mm/fault.c +++ b/arch/frv/mm/fault.c | |||
@@ -19,9 +19,9 @@ | |||
19 | #include <linux/kernel.h> | 19 | #include <linux/kernel.h> |
20 | #include <linux/ptrace.h> | 20 | #include <linux/ptrace.h> |
21 | #include <linux/hardirq.h> | 21 | #include <linux/hardirq.h> |
22 | #include <linux/uaccess.h> | ||
22 | 23 | ||
23 | #include <asm/pgtable.h> | 24 | #include <asm/pgtable.h> |
24 | #include <asm/uaccess.h> | ||
25 | #include <asm/gdb-stub.h> | 25 | #include <asm/gdb-stub.h> |
26 | 26 | ||
27 | /*****************************************************************************/ | 27 | /*****************************************************************************/ |
@@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear | |||
78 | * If we're in an interrupt or have no user | 78 | * If we're in an interrupt or have no user |
79 | * context, we must not take the fault.. | 79 | * context, we must not take the fault.. |
80 | */ | 80 | */ |
81 | if (in_atomic() || !mm) | 81 | if (faulthandler_disabled() || !mm) |
82 | goto no_context; | 82 | goto no_context; |
83 | 83 | ||
84 | if (user_mode(__frame)) | 84 | if (user_mode(__frame)) |
diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index ba5ba7accd0d..70b40d1205a6 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c | |||
@@ -11,10 +11,10 @@ | |||
11 | #include <linux/kprobes.h> | 11 | #include <linux/kprobes.h> |
12 | #include <linux/kdebug.h> | 12 | #include <linux/kdebug.h> |
13 | #include <linux/prefetch.h> | 13 | #include <linux/prefetch.h> |
14 | #include <linux/uaccess.h> | ||
14 | 15 | ||
15 | #include <asm/pgtable.h> | 16 | #include <asm/pgtable.h> |
16 | #include <asm/processor.h> | 17 | #include <asm/processor.h> |
17 | #include <asm/uaccess.h> | ||
18 | 18 | ||
19 | extern int die(char *, struct pt_regs *, long); | 19 | extern int die(char *, struct pt_regs *, long); |
20 | 20 | ||
@@ -96,7 +96,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re | |||
96 | /* | 96 | /* |
97 | * If we're in an interrupt or have no user context, we must not take the fault.. | 97 | * If we're in an interrupt or have no user context, we must not take the fault.. |
98 | */ | 98 | */ |
99 | if (in_atomic() || !mm) | 99 | if (faulthandler_disabled() || !mm) |
100 | goto no_context; | 100 | goto no_context; |
101 | 101 | ||
102 | #ifdef CONFIG_VIRTUAL_MEM_MAP | 102 | #ifdef CONFIG_VIRTUAL_MEM_MAP |
diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c index e3d4d4890104..8f9875b7933d 100644 --- a/arch/m32r/mm/fault.c +++ b/arch/m32r/mm/fault.c | |||
@@ -24,9 +24,9 @@ | |||
24 | #include <linux/vt_kern.h> /* For unblank_screen() */ | 24 | #include <linux/vt_kern.h> /* For unblank_screen() */ |
25 | #include <linux/highmem.h> | 25 | #include <linux/highmem.h> |
26 | #include <linux/module.h> | 26 | #include <linux/module.h> |
27 | #include <linux/uaccess.h> | ||
27 | 28 | ||
28 | #include <asm/m32r.h> | 29 | #include <asm/m32r.h> |
29 | #include <asm/uaccess.h> | ||
30 | #include <asm/hardirq.h> | 30 | #include <asm/hardirq.h> |
31 | #include <asm/mmu_context.h> | 31 | #include <asm/mmu_context.h> |
32 | #include <asm/tlbflush.h> | 32 | #include <asm/tlbflush.h> |
@@ -111,10 +111,10 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, | |||
111 | mm = tsk->mm; | 111 | mm = tsk->mm; |
112 | 112 | ||
113 | /* | 113 | /* |
114 | * If we're in an interrupt or have no user context or are running in an | 114 | * If we're in an interrupt or have no user context or have pagefaults |
115 | * atomic region then we must not take the fault.. | 115 | * disabled then we must not take the fault. |
116 | */ | 116 | */ |
117 | if (in_atomic() || !mm) | 117 | if (faulthandler_disabled() || !mm) |
118 | goto bad_area_nosemaphore; | 118 | goto bad_area_nosemaphore; |
119 | 119 | ||
120 | if (error_code & ACE_USERMODE) | 120 | if (error_code & ACE_USERMODE) |
diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index b2f04aee46ec..6a94cdd0c830 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c | |||
@@ -10,10 +10,10 @@ | |||
10 | #include <linux/ptrace.h> | 10 | #include <linux/ptrace.h> |
11 | #include <linux/interrupt.h> | 11 | #include <linux/interrupt.h> |
12 | #include <linux/module.h> | 12 | #include <linux/module.h> |
13 | #include <linux/uaccess.h> | ||
13 | 14 | ||
14 | #include <asm/setup.h> | 15 | #include <asm/setup.h> |
15 | #include <asm/traps.h> | 16 | #include <asm/traps.h> |
16 | #include <asm/uaccess.h> | ||
17 | #include <asm/pgalloc.h> | 17 | #include <asm/pgalloc.h> |
18 | 18 | ||
19 | extern void die_if_kernel(char *, struct pt_regs *, long); | 19 | extern void die_if_kernel(char *, struct pt_regs *, long); |
@@ -81,7 +81,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, | |||
81 | * If we're in an interrupt or have no user | 81 | * If we're in an interrupt or have no user |
82 | * context, we must not take the fault.. | 82 | * context, we must not take the fault.. |
83 | */ | 83 | */ |
84 | if (in_atomic() || !mm) | 84 | if (faulthandler_disabled() || !mm) |
85 | goto no_context; | 85 | goto no_context; |
86 | 86 | ||
87 | if (user_mode(regs)) | 87 | if (user_mode(regs)) |
diff --git a/arch/metag/mm/fault.c b/arch/metag/mm/fault.c index 2de5dc695a87..f57edca63609 100644 --- a/arch/metag/mm/fault.c +++ b/arch/metag/mm/fault.c | |||
@@ -105,7 +105,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, | |||
105 | 105 | ||
106 | mm = tsk->mm; | 106 | mm = tsk->mm; |
107 | 107 | ||
108 | if (in_atomic() || !mm) | 108 | if (faulthandler_disabled() || !mm) |
109 | goto no_context; | 109 | goto no_context; |
110 | 110 | ||
111 | if (user_mode(regs)) | 111 | if (user_mode(regs)) |
diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index d46a5ebb7570..177dfc003643 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c | |||
@@ -107,14 +107,14 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, | |||
107 | if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) | 107 | if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) |
108 | is_write = 0; | 108 | is_write = 0; |
109 | 109 | ||
110 | if (unlikely(in_atomic() || !mm)) { | 110 | if (unlikely(faulthandler_disabled() || !mm)) { |
111 | if (kernel_mode(regs)) | 111 | if (kernel_mode(regs)) |
112 | goto bad_area_nosemaphore; | 112 | goto bad_area_nosemaphore; |
113 | 113 | ||
114 | /* in_atomic() in user mode is really bad, | 114 | /* faulthandler_disabled() in user mode is really bad, |
115 | as is current->mm == NULL. */ | 115 | as is current->mm == NULL. */ |
116 | pr_emerg("Page fault in user mode with in_atomic(), mm = %p\n", | 116 | pr_emerg("Page fault in user mode with faulthandler_disabled(), mm = %p\n", |
117 | mm); | 117 | mm); |
118 | pr_emerg("r15 = %lx MSR = %lx\n", | 118 | pr_emerg("r15 = %lx MSR = %lx\n", |
119 | regs->r15, regs->msr); | 119 | regs->r15, regs->msr); |
120 | die("Weird page fault", regs, SIGSEGV); | 120 | die("Weird page fault", regs, SIGSEGV); |
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 7ff8637e530d..36c0f26fac6b 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c | |||
@@ -21,10 +21,10 @@ | |||
21 | #include <linux/module.h> | 21 | #include <linux/module.h> |
22 | #include <linux/kprobes.h> | 22 | #include <linux/kprobes.h> |
23 | #include <linux/perf_event.h> | 23 | #include <linux/perf_event.h> |
24 | #include <linux/uaccess.h> | ||
24 | 25 | ||
25 | #include <asm/branch.h> | 26 | #include <asm/branch.h> |
26 | #include <asm/mmu_context.h> | 27 | #include <asm/mmu_context.h> |
27 | #include <asm/uaccess.h> | ||
28 | #include <asm/ptrace.h> | 28 | #include <asm/ptrace.h> |
29 | #include <asm/highmem.h> /* For VMALLOC_END */ | 29 | #include <asm/highmem.h> /* For VMALLOC_END */ |
30 | #include <linux/kdebug.h> | 30 | #include <linux/kdebug.h> |
@@ -94,7 +94,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, | |||
94 | * If we're in an interrupt or have no user | 94 | * If we're in an interrupt or have no user |
95 | * context, we must not take the fault.. | 95 | * context, we must not take the fault.. |
96 | */ | 96 | */ |
97 | if (in_atomic() || !mm) | 97 | if (faulthandler_disabled() || !mm) |
98 | goto bad_area_nosemaphore; | 98 | goto bad_area_nosemaphore; |
99 | 99 | ||
100 | if (user_mode(regs)) | 100 | if (user_mode(regs)) |
diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c index 0c2cc5d39c8e..4a1d181ed32f 100644 --- a/arch/mn10300/mm/fault.c +++ b/arch/mn10300/mm/fault.c | |||
@@ -23,8 +23,8 @@ | |||
23 | #include <linux/interrupt.h> | 23 | #include <linux/interrupt.h> |
24 | #include <linux/init.h> | 24 | #include <linux/init.h> |
25 | #include <linux/vt_kern.h> /* For unblank_screen() */ | 25 | #include <linux/vt_kern.h> /* For unblank_screen() */ |
26 | #include <linux/uaccess.h> | ||
26 | 27 | ||
27 | #include <asm/uaccess.h> | ||
28 | #include <asm/pgalloc.h> | 28 | #include <asm/pgalloc.h> |
29 | #include <asm/hardirq.h> | 29 | #include <asm/hardirq.h> |
30 | #include <asm/cpu-regs.h> | 30 | #include <asm/cpu-regs.h> |
@@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code, | |||
168 | * If we're in an interrupt or have no user | 168 | * If we're in an interrupt or have no user |
169 | * context, we must not take the fault.. | 169 | * context, we must not take the fault.. |
170 | */ | 170 | */ |
171 | if (in_atomic() || !mm) | 171 | if (faulthandler_disabled() || !mm) |
172 | goto no_context; | 172 | goto no_context; |
173 | 173 | ||
174 | if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) | 174 | if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) |
diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c index 0c9b6afe69e9..b51878b0c6b8 100644 --- a/arch/nios2/mm/fault.c +++ b/arch/nios2/mm/fault.c | |||
@@ -77,7 +77,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, | |||
77 | * If we're in an interrupt or have no user | 77 | * If we're in an interrupt or have no user |
78 | * context, we must not take the fault.. | 78 | * context, we must not take the fault.. |
79 | */ | 79 | */ |
80 | if (in_atomic() || !mm) | 80 | if (faulthandler_disabled() || !mm) |
81 | goto bad_area_nosemaphore; | 81 | goto bad_area_nosemaphore; |
82 | 82 | ||
83 | if (user_mode(regs)) | 83 | if (user_mode(regs)) |
diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index 47ee620d15d2..6548fd1d2e62 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c | |||
@@ -26,9 +26,9 @@ | |||
26 | #include <linux/console.h> | 26 | #include <linux/console.h> |
27 | #include <linux/bug.h> | 27 | #include <linux/bug.h> |
28 | #include <linux/ratelimit.h> | 28 | #include <linux/ratelimit.h> |
29 | #include <linux/uaccess.h> | ||
29 | 30 | ||
30 | #include <asm/assembly.h> | 31 | #include <asm/assembly.h> |
31 | #include <asm/uaccess.h> | ||
32 | #include <asm/io.h> | 32 | #include <asm/io.h> |
33 | #include <asm/irq.h> | 33 | #include <asm/irq.h> |
34 | #include <asm/traps.h> | 34 | #include <asm/traps.h> |
@@ -800,7 +800,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs) | |||
800 | * unless pagefault_disable() was called before. | 800 | * unless pagefault_disable() was called before. |
801 | */ | 801 | */ |
802 | 802 | ||
803 | if (fault_space == 0 && !in_atomic()) | 803 | if (fault_space == 0 && !faulthandler_disabled()) |
804 | { | 804 | { |
805 | pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); | 805 | pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); |
806 | parisc_terminate("Kernel Fault", regs, code, fault_address); | 806 | parisc_terminate("Kernel Fault", regs, code, fault_address); |
diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index e5120e653240..15503adddf4f 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c | |||
@@ -15,8 +15,8 @@ | |||
15 | #include <linux/sched.h> | 15 | #include <linux/sched.h> |
16 | #include <linux/interrupt.h> | 16 | #include <linux/interrupt.h> |
17 | #include <linux/module.h> | 17 | #include <linux/module.h> |
18 | #include <linux/uaccess.h> | ||
18 | 19 | ||
19 | #include <asm/uaccess.h> | ||
20 | #include <asm/traps.h> | 20 | #include <asm/traps.h> |
21 | 21 | ||
22 | /* Various important other fields */ | 22 | /* Various important other fields */ |
@@ -207,7 +207,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, | |||
207 | int fault; | 207 | int fault; |
208 | unsigned int flags; | 208 | unsigned int flags; |
209 | 209 | ||
210 | if (in_atomic()) | 210 | if (pagefault_disabled()) |
211 | goto no_context; | 211 | goto no_context; |
212 | 212 | ||
213 | tsk = current; | 213 | tsk = current; |
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index b396868d2aa7..6d535973b200 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c | |||
@@ -33,13 +33,13 @@ | |||
33 | #include <linux/ratelimit.h> | 33 | #include <linux/ratelimit.h> |
34 | #include <linux/context_tracking.h> | 34 | #include <linux/context_tracking.h> |
35 | #include <linux/hugetlb.h> | 35 | #include <linux/hugetlb.h> |
36 | #include <linux/uaccess.h> | ||
36 | 37 | ||
37 | #include <asm/firmware.h> | 38 | #include <asm/firmware.h> |
38 | #include <asm/page.h> | 39 | #include <asm/page.h> |
39 | #include <asm/pgtable.h> | 40 | #include <asm/pgtable.h> |
40 | #include <asm/mmu.h> | 41 | #include <asm/mmu.h> |
41 | #include <asm/mmu_context.h> | 42 | #include <asm/mmu_context.h> |
42 | #include <asm/uaccess.h> | ||
43 | #include <asm/tlbflush.h> | 43 | #include <asm/tlbflush.h> |
44 | #include <asm/siginfo.h> | 44 | #include <asm/siginfo.h> |
45 | #include <asm/debug.h> | 45 | #include <asm/debug.h> |
@@ -272,15 +272,16 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, | |||
272 | if (!arch_irq_disabled_regs(regs)) | 272 | if (!arch_irq_disabled_regs(regs)) |
273 | local_irq_enable(); | 273 | local_irq_enable(); |
274 | 274 | ||
275 | if (in_atomic() || mm == NULL) { | 275 | if (faulthandler_disabled() || mm == NULL) { |
276 | if (!user_mode(regs)) { | 276 | if (!user_mode(regs)) { |
277 | rc = SIGSEGV; | 277 | rc = SIGSEGV; |
278 | goto bail; | 278 | goto bail; |
279 | } | 279 | } |
280 | /* in_atomic() in user mode is really bad, | 280 | /* faulthandler_disabled() in user mode is really bad, |
281 | as is current->mm == NULL. */ | 281 | as is current->mm == NULL. */ |
282 | printk(KERN_EMERG "Page fault in user mode with " | 282 | printk(KERN_EMERG "Page fault in user mode with " |
283 | "in_atomic() = %d mm = %p\n", in_atomic(), mm); | 283 | "faulthandler_disabled() = %d mm = %p\n", |
284 | faulthandler_disabled(), mm); | ||
284 | printk(KERN_EMERG "NIP = %lx MSR = %lx\n", | 285 | printk(KERN_EMERG "NIP = %lx MSR = %lx\n", |
285 | regs->nip, regs->msr); | 286 | regs->nip, regs->msr); |
286 | die("Weird page fault", regs, SIGSEGV); | 287 | die("Weird page fault", regs, SIGSEGV); |
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 76515bcea2f1..4c8f5d7f9c23 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c | |||
@@ -399,7 +399,7 @@ static inline int do_exception(struct pt_regs *regs, int access) | |||
399 | * user context. | 399 | * user context. |
400 | */ | 400 | */ |
401 | fault = VM_FAULT_BADCONTEXT; | 401 | fault = VM_FAULT_BADCONTEXT; |
402 | if (unlikely(!user_space_fault(regs) || in_atomic() || !mm)) | 402 | if (unlikely(!user_space_fault(regs) || faulthandler_disabled() || !mm)) |
403 | goto out; | 403 | goto out; |
404 | 404 | ||
405 | address = trans_exc_code & __FAIL_ADDR_MASK; | 405 | address = trans_exc_code & __FAIL_ADDR_MASK; |
diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c index 6860beb2a280..37a6c2e0e969 100644 --- a/arch/score/mm/fault.c +++ b/arch/score/mm/fault.c | |||
@@ -34,6 +34,7 @@ | |||
34 | #include <linux/string.h> | 34 | #include <linux/string.h> |
35 | #include <linux/types.h> | 35 | #include <linux/types.h> |
36 | #include <linux/ptrace.h> | 36 | #include <linux/ptrace.h> |
37 | #include <linux/uaccess.h> | ||
37 | 38 | ||
38 | /* | 39 | /* |
39 | * This routine handles page faults. It determines the address, | 40 | * This routine handles page faults. It determines the address, |
@@ -73,7 +74,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, | |||
73 | * If we're in an interrupt or have no user | 74 | * If we're in an interrupt or have no user |
74 | * context, we must not take the fault.. | 75 | * context, we must not take the fault.. |
75 | */ | 76 | */ |
76 | if (in_atomic() || !mm) | 77 | if (pagefault_disabled() || !mm) |
77 | goto bad_area_nosemaphore; | 78 | goto bad_area_nosemaphore; |
78 | 79 | ||
79 | if (user_mode(regs)) | 80 | if (user_mode(regs)) |
diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index a58fec9b55e0..79d8276377d1 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c | |||
@@ -17,6 +17,7 @@ | |||
17 | #include <linux/kprobes.h> | 17 | #include <linux/kprobes.h> |
18 | #include <linux/perf_event.h> | 18 | #include <linux/perf_event.h> |
19 | #include <linux/kdebug.h> | 19 | #include <linux/kdebug.h> |
20 | #include <linux/uaccess.h> | ||
20 | #include <asm/io_trapped.h> | 21 | #include <asm/io_trapped.h> |
21 | #include <asm/mmu_context.h> | 22 | #include <asm/mmu_context.h> |
22 | #include <asm/tlbflush.h> | 23 | #include <asm/tlbflush.h> |
@@ -438,9 +439,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, | |||
438 | 439 | ||
439 | /* | 440 | /* |
440 | * If we're in an interrupt, have no user context or are running | 441 | * If we're in an interrupt, have no user context or are running |
441 | * in an atomic region then we must not take the fault: | 442 | * with pagefaults disabled then we must not take the fault: |
442 | */ | 443 | */ |
443 | if (unlikely(in_atomic() || !mm)) { | 444 | if (unlikely(faulthandler_disabled() || !mm)) { |
444 | bad_area_nosemaphore(regs, error_code, address); | 445 | bad_area_nosemaphore(regs, error_code, address); |
445 | return; | 446 | return; |
446 | } | 447 | } |
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 70d817154fe8..c399e7b3b035 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c | |||
@@ -21,6 +21,7 @@ | |||
21 | #include <linux/perf_event.h> | 21 | #include <linux/perf_event.h> |
22 | #include <linux/interrupt.h> | 22 | #include <linux/interrupt.h> |
23 | #include <linux/kdebug.h> | 23 | #include <linux/kdebug.h> |
24 | #include <linux/uaccess.h> | ||
24 | 25 | ||
25 | #include <asm/page.h> | 26 | #include <asm/page.h> |
26 | #include <asm/pgtable.h> | 27 | #include <asm/pgtable.h> |
@@ -29,7 +30,6 @@ | |||
29 | #include <asm/setup.h> | 30 | #include <asm/setup.h> |
30 | #include <asm/smp.h> | 31 | #include <asm/smp.h> |
31 | #include <asm/traps.h> | 32 | #include <asm/traps.h> |
32 | #include <asm/uaccess.h> | ||
33 | 33 | ||
34 | #include "mm_32.h" | 34 | #include "mm_32.h" |
35 | 35 | ||
@@ -196,7 +196,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, | |||
196 | * If we're in an interrupt or have no user | 196 | * If we're in an interrupt or have no user |
197 | * context, we must not take the fault.. | 197 | * context, we must not take the fault.. |
198 | */ | 198 | */ |
199 | if (in_atomic() || !mm) | 199 | if (pagefault_disabled() || !mm) |
200 | goto no_context; | 200 | goto no_context; |
201 | 201 | ||
202 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); | 202 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 479823249429..e9268ea1a68d 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c | |||
@@ -22,12 +22,12 @@ | |||
22 | #include <linux/kdebug.h> | 22 | #include <linux/kdebug.h> |
23 | #include <linux/percpu.h> | 23 | #include <linux/percpu.h> |
24 | #include <linux/context_tracking.h> | 24 | #include <linux/context_tracking.h> |
25 | #include <linux/uaccess.h> | ||
25 | 26 | ||
26 | #include <asm/page.h> | 27 | #include <asm/page.h> |
27 | #include <asm/pgtable.h> | 28 | #include <asm/pgtable.h> |
28 | #include <asm/openprom.h> | 29 | #include <asm/openprom.h> |
29 | #include <asm/oplib.h> | 30 | #include <asm/oplib.h> |
30 | #include <asm/uaccess.h> | ||
31 | #include <asm/asi.h> | 31 | #include <asm/asi.h> |
32 | #include <asm/lsu.h> | 32 | #include <asm/lsu.h> |
33 | #include <asm/sections.h> | 33 | #include <asm/sections.h> |
@@ -330,7 +330,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) | |||
330 | * If we're in an interrupt or have no user | 330 | * If we're in an interrupt or have no user |
331 | * context, we must not take the fault.. | 331 | * context, we must not take the fault.. |
332 | */ | 332 | */ |
333 | if (in_atomic() || !mm) | 333 | if (faulthandler_disabled() || !mm) |
334 | goto intr_or_no_mm; | 334 | goto intr_or_no_mm; |
335 | 335 | ||
336 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); | 336 | perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); |
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 4ca0d6ba5ec8..cee9b77ddd05 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c | |||
@@ -2706,7 +2706,7 @@ void hugetlb_setup(struct pt_regs *regs) | |||
2706 | struct mm_struct *mm = current->mm; | 2706 | struct mm_struct *mm = current->mm; |
2707 | struct tsb_config *tp; | 2707 | struct tsb_config *tp; |
2708 | 2708 | ||
2709 | if (in_atomic() || !mm) { | 2709 | if (faulthandler_disabled() || !mm) { |
2710 | const struct exception_table_entry *entry; | 2710 | const struct exception_table_entry *entry; |
2711 | 2711 | ||
2712 | entry = search_exception_tables(regs->tpc); | 2712 | entry = search_exception_tables(regs->tpc); |
diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c index e83cc999da02..3f4f58d34a92 100644 --- a/arch/tile/mm/fault.c +++ b/arch/tile/mm/fault.c | |||
@@ -354,9 +354,9 @@ static int handle_page_fault(struct pt_regs *regs, | |||
354 | 354 | ||
355 | /* | 355 | /* |
356 | * If we're in an interrupt, have no user context or are running in an | 356 | * If we're in an interrupt, have no user context or are running in an |
357 | * atomic region then we must not take the fault. | 357 | * region with pagefaults disabled then we must not take the fault. |
358 | */ | 358 | */ |
359 | if (in_atomic() || !mm) { | 359 | if (pagefault_disabled() || !mm) { |
360 | vma = NULL; /* happy compiler */ | 360 | vma = NULL; /* happy compiler */ |
361 | goto bad_area_nosemaphore; | 361 | goto bad_area_nosemaphore; |
362 | } | 362 | } |
diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 8e4daf44e980..f9c9e5a6beba 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c | |||
@@ -35,10 +35,10 @@ int handle_page_fault(unsigned long address, unsigned long ip, | |||
35 | *code_out = SEGV_MAPERR; | 35 | *code_out = SEGV_MAPERR; |
36 | 36 | ||
37 | /* | 37 | /* |
38 | * If the fault was during atomic operation, don't take the fault, just | 38 | * If the fault was with pagefaults disabled, don't take the fault, just |
39 | * fail. | 39 | * fail. |
40 | */ | 40 | */ |
41 | if (in_atomic()) | 41 | if (faulthandler_disabled()) |
42 | goto out_nosemaphore; | 42 | goto out_nosemaphore; |
43 | 43 | ||
44 | if (is_user) | 44 | if (is_user) |
diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c index 0dc922dba915..afccef5529cc 100644 --- a/arch/unicore32/mm/fault.c +++ b/arch/unicore32/mm/fault.c | |||
@@ -218,7 +218,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs) | |||
218 | * If we're in an interrupt or have no user | 218 | * If we're in an interrupt or have no user |
219 | * context, we must not take the fault.. | 219 | * context, we must not take the fault.. |
220 | */ | 220 | */ |
221 | if (in_atomic() || !mm) | 221 | if (faulthandler_disabled() || !mm) |
222 | goto no_context; | 222 | goto no_context; |
223 | 223 | ||
224 | if (user_mode(regs)) | 224 | if (user_mode(regs)) |
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 181c53bac3a7..9dc909841739 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c | |||
@@ -13,6 +13,7 @@ | |||
13 | #include <linux/hugetlb.h> /* hstate_index_to_shift */ | 13 | #include <linux/hugetlb.h> /* hstate_index_to_shift */ |
14 | #include <linux/prefetch.h> /* prefetchw */ | 14 | #include <linux/prefetch.h> /* prefetchw */ |
15 | #include <linux/context_tracking.h> /* exception_enter(), ... */ | 15 | #include <linux/context_tracking.h> /* exception_enter(), ... */ |
16 | #include <linux/uaccess.h> /* faulthandler_disabled() */ | ||
16 | 17 | ||
17 | #include <asm/traps.h> /* dotraplinkage, ... */ | 18 | #include <asm/traps.h> /* dotraplinkage, ... */ |
18 | #include <asm/pgalloc.h> /* pgd_*(), ... */ | 19 | #include <asm/pgalloc.h> /* pgd_*(), ... */ |
@@ -1126,9 +1127,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code, | |||
1126 | 1127 | ||
1127 | /* | 1128 | /* |
1128 | * If we're in an interrupt, have no user context or are running | 1129 | * If we're in an interrupt, have no user context or are running |
1129 | * in an atomic region then we must not take the fault: | 1130 | * in a region with pagefaults disabled then we must not take the fault |
1130 | */ | 1131 | */ |
1131 | if (unlikely(in_atomic() || !mm)) { | 1132 | if (unlikely(faulthandler_disabled() || !mm)) { |
1132 | bad_area_nosemaphore(regs, error_code, address); | 1133 | bad_area_nosemaphore(regs, error_code, address); |
1133 | return; | 1134 | return; |
1134 | } | 1135 | } |
diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 9e3571a6535c..83a44a33cfa1 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c | |||
@@ -15,10 +15,10 @@ | |||
15 | #include <linux/mm.h> | 15 | #include <linux/mm.h> |
16 | #include <linux/module.h> | 16 | #include <linux/module.h> |
17 | #include <linux/hardirq.h> | 17 | #include <linux/hardirq.h> |
18 | #include <linux/uaccess.h> | ||
18 | #include <asm/mmu_context.h> | 19 | #include <asm/mmu_context.h> |
19 | #include <asm/cacheflush.h> | 20 | #include <asm/cacheflush.h> |
20 | #include <asm/hardirq.h> | 21 | #include <asm/hardirq.h> |
21 | #include <asm/uaccess.h> | ||
22 | #include <asm/pgalloc.h> | 22 | #include <asm/pgalloc.h> |
23 | 23 | ||
24 | DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST; | 24 | DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST; |
@@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs) | |||
57 | /* If we're in an interrupt or have no user | 57 | /* If we're in an interrupt or have no user |
58 | * context, we must not take the fault.. | 58 | * context, we must not take the fault.. |
59 | */ | 59 | */ |
60 | if (in_atomic() || !mm) { | 60 | if (faulthandler_disabled() || !mm) { |
61 | bad_page_fault(regs, address, SIGSEGV); | 61 | bad_page_fault(regs, address, SIGSEGV); |
62 | return; | 62 | return; |
63 | } | 63 | } |
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 23290cc93a24..90786d2d74e5 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h | |||
@@ -59,6 +59,18 @@ static inline void pagefault_enable(void) | |||
59 | */ | 59 | */ |
60 | #define pagefault_disabled() (current->pagefault_disabled != 0) | 60 | #define pagefault_disabled() (current->pagefault_disabled != 0) |
61 | 61 | ||
62 | /* | ||
63 | * The pagefault handler is in general disabled by pagefault_disable() or | ||
64 | * when in irq context (via in_atomic()). | ||
65 | * | ||
66 | * This function should only be used by the fault handlers. Other users should | ||
67 | * stick to pagefault_disabled(). | ||
68 | * Please NEVER use preempt_disable() to disable the fault handler. With | ||
69 | * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. | ||
70 | * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. | ||
71 | */ | ||
72 | #define faulthandler_disabled() (pagefault_disabled() || in_atomic()) | ||
73 | |||
62 | #ifndef ARCH_HAS_NOCACHE_UACCESS | 74 | #ifndef ARCH_HAS_NOCACHE_UACCESS |
63 | 75 | ||
64 | static inline unsigned long __copy_from_user_inatomic_nocache(void *to, | 76 | static inline unsigned long __copy_from_user_inatomic_nocache(void *to, |