1 files changed, 263 insertions, 183 deletions
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index c76ef1d701c9..8c3f3113a6ec 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -26,6 +26,7 @@
 #include <linux/kprobes.h>
 #include <linux/uaccess.h>
 #include <linux/kdebug.h>
+#include <linux/magic.h>
 #include <asm/system.h>
 #include <asm/desc.h>
@@ -91,8 +92,8 @@ static inline int notify_page_fault(struct pt_regs *regs)
 *
 * Opcode checker based on code by Richard Brunner
 */
-static int is_prefetch(struct pt_regs *regs, unsigned long addr,
+static int is_prefetch(struct pt_regs *regs, unsigned long error_code,
-                       unsigned long error_code)
+                        unsigned long addr)
 {
        unsigned char *instr;
        int scan_more = 1;
@@ -409,15 +410,15 @@ static void show_fault_oops(struct pt_regs *regs, unsigned long error_code,
 }
 #ifdef CONFIG_X86_64
-static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
+static noinline void pgtable_bad(struct pt_regs *regs,
-                                 unsigned long error_code)
+                         unsigned long error_code, unsigned long address)
 {
        unsigned long flags = oops_begin();
        int sig = SIGKILL;
-        struct task_struct *tsk;
+        struct task_struct *tsk = current;
        printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
-               current->comm, address);
+               tsk->comm, address);
        dump_pagetable(address);
        tsk = current;
        tsk->thread.cr2 = address;
@@ -429,6 +430,196 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 }
 #endif
+static noinline void no_context(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        struct task_struct *tsk = current;
+        unsigned long *stackend;
+#ifdef CONFIG_X86_64
+        unsigned long flags;
+        int sig;
+#endif
+        /* Are we prepared to handle this kernel fault?  */
+        if (fixup_exception(regs))
+                return;
+        /*
+         * X86_32
+         * Valid to do another page fault here, because if this fault
+         * had been triggered by is_prefetch fixup_exception would have
+         * handled it.
+         *
+         * X86_64
+         * Hall of shame of CPU/BIOS bugs.
+         */
+        if (is_prefetch(regs, error_code, address))
+                return;
+        if (is_errata93(regs, address))
+                return;
+        /*
+         * Oops. The kernel tried to access some bad page. We'll have to
+         * terminate things with extreme prejudice.
+         */
+#ifdef CONFIG_X86_32
+        bust_spinlocks(1);
+#else
+        flags = oops_begin();
+#endif
+        show_fault_oops(regs, error_code, address);
+        stackend = end_of_stack(tsk);
+        if (*stackend != STACK_END_MAGIC)
+                printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
+        tsk->thread.cr2 = address;
+        tsk->thread.trap_no = 14;
+        tsk->thread.error_code = error_code;
+#ifdef CONFIG_X86_32
+        die("Oops", regs, error_code);
+        bust_spinlocks(0);
+        do_exit(SIGKILL);
+#else
+        sig = SIGKILL;
+        if (__die("Oops", regs, error_code))
+                sig = 0;
+        /* Executive summary in case the body of the oops scrolled away */
+        printk(KERN_EMERG "CR2: %016lx\n", address);
+        oops_end(flags, regs, sig);
+#endif
+}
+static void __bad_area_nosemaphore(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address,
+                        int si_code)
+{
+        struct task_struct *tsk = current;
+        /* User mode accesses just cause a SIGSEGV */
+        if (error_code & PF_USER) {
+                /*
+                 * It's possible to have interrupts off here.
+                 */
+                local_irq_enable();
+                /*
+                 * Valid to do another page fault here because this one came
+                 * from user space.
+                 */
+                if (is_prefetch(regs, error_code, address))
+                        return;
+                if (is_errata100(regs, address))
+                        return;
+                if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
+                    printk_ratelimit()) {
+                        printk(
+                        "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
+                        task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
+                        tsk->comm, task_pid_nr(tsk), address,
+                        (void *) regs->ip, (void *) regs->sp, error_code);
+                        print_vma_addr(" in ", regs->ip);
+                        printk("\n");
+                }
+                tsk->thread.cr2 = address;
+                /* Kernel addresses are always protection faults */
+                tsk->thread.error_code = error_code | (address >= TASK_SIZE);
+                tsk->thread.trap_no = 14;
+                force_sig_info_fault(SIGSEGV, si_code, address, tsk);
+                return;
+        }
+        if (is_f00f_bug(regs, address))
+                return;
+        no_context(regs, error_code, address);
+}
+static noinline void bad_area_nosemaphore(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
+}
+static void __bad_area(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address,
+                        int si_code)
+{
+        struct mm_struct *mm = current->mm;
+        /*
+         * Something tried to access memory that isn't in our memory map..
+         * Fix it, but check if it's kernel or user first..
+         */
+        up_read(&mm->mmap_sem);
+        __bad_area_nosemaphore(regs, error_code, address, si_code);
+}
+static noinline void bad_area(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area(regs, error_code, address, SEGV_MAPERR);
+}
+static noinline void bad_area_access_error(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area(regs, error_code, address, SEGV_ACCERR);
+}
+/* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */
+static void out_of_memory(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        /*
+         * We ran out of memory, call the OOM killer, and return the userspace
+         * (which will retry the fault, or kill us if we got oom-killed).
+         */
+        up_read(&current->mm->mmap_sem);
+        pagefault_out_of_memory();
+}
+static void do_sigbus(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        struct task_struct *tsk = current;
+        struct mm_struct *mm = tsk->mm;
+        up_read(&mm->mmap_sem);
+        /* Kernel mode? Handle exceptions or die */
+        if (!(error_code & PF_USER))
+                no_context(regs, error_code, address);
+#ifdef CONFIG_X86_32
+        /* User space => ok to do another page fault */
+        if (is_prefetch(regs, error_code, address))
+                return;
+#endif
+        tsk->thread.cr2 = address;
+        tsk->thread.error_code = error_code;
+        tsk->thread.trap_no = 14;
+        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
+}
+static noinline void mm_fault_error(struct pt_regs *regs,
+                unsigned long error_code, unsigned long address, unsigned int fault)
+{
+        if (fault & VM_FAULT_OOM)
+                out_of_memory(regs, error_code, address);
+        else if (fault & VM_FAULT_SIGBUS)
+                do_sigbus(regs, error_code, address);
+        else
+                BUG();
+}
 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 {
        if ((error_code & PF_WRITE) && !pte_write(*pte))
@@ -448,8 +639,8 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 * There are no security implications to leaving a stale TLB when
 * increasing the permissions on a page.
 */
-static int spurious_fault(unsigned long address,
+static noinline int spurious_fault(unsigned long error_code,
-                          unsigned long error_code)
+                                unsigned long address)
 {
        pgd_t *pgd;
        pud_t *pud;
@@ -494,7 +685,7 @@ static int spurious_fault(unsigned long address,
 *
 * This assumes no large pages in there.
 */
-static int vmalloc_fault(unsigned long address)
+static noinline int vmalloc_fault(unsigned long address)
 {
 #ifdef CONFIG_X86_32
        unsigned long pgd_paddr;
@@ -573,6 +764,25 @@ static int vmalloc_fault(unsigned long address)
 int show_unhandled_signals = 1;
+static inline int access_error(unsigned long error_code, int write,
+                                struct vm_area_struct *vma)
+{
+        if (write) {
+                /* write, present and write, not present */
+                if (unlikely(!(vma->vm_flags & VM_WRITE)))
+                        return 1;
+        } else if (unlikely(error_code & PF_PROT)) {
+                /* read, present */
+                return 1;
+        } else {
+                /* read, not present */
+                if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
+                        return 1;
+        }
+        return 0;
+}
 /*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
@@ -583,16 +793,12 @@ asmlinkage
 #endif
 void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 {
+        unsigned long address;
        struct task_struct *tsk;
        struct mm_struct *mm;
        struct vm_area_struct *vma;
-        unsigned long address;
+        int write;
-        int write, si_code;
        int fault;
-#ifdef CONFIG_X86_64
-        unsigned long flags;
-        int sig;
-#endif
        tsk = current;
        mm = tsk->mm;
@@ -601,8 +807,6 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
        /* get the address */
        address = read_cr2();
-        si_code = SEGV_MAPERR;
        if (unlikely(kmmio_fault(regs, address)))
                return;
@@ -629,7 +833,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                        return;
                /* Can handle a stale RO->RW TLB */
-                if (spurious_fault(address, error_code))
+                if (spurious_fault(error_code, address))
                        return;
                /* kprobes don't want to hook the spurious faults. */
@@ -639,13 +843,12 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                 * Don't take the mm semaphore here. If we fixup a prefetch
                 * fault we could otherwise deadlock.
                 */
-                goto bad_area_nosemaphore;
+                bad_area_nosemaphore(regs, error_code, address);
+                return;
        }
-        /* kprobes don't want to hook the spurious faults. */
+        if (unlikely(notify_page_fault(regs)))
-        if (notify_page_fault(regs))
                return;
        /*
         * It's safe to allow irq's after cr2 has been saved and the
         * vmalloc fault has been handled.
@@ -661,15 +864,17 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 #ifdef CONFIG_X86_64
        if (unlikely(error_code & PF_RSVD))
-                pgtable_bad(address, regs, error_code);
+                pgtable_bad(regs, error_code, address);
 #endif
        /*
         * If we're in an interrupt, have no user context or are running in an
         * atomic region then we must not take the fault.
         */
-        if (unlikely(in_atomic() || !mm))
+        if (unlikely(in_atomic() || !mm)) {
-                goto bad_area_nosemaphore;
+                bad_area_nosemaphore(regs, error_code, address);
+                return;
+        }
        /*
         * When running in the kernel we expect faults to occur only to
@@ -687,20 +892,26 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
         * source.  If this is invalid we can skip the address space check,
         * thus avoiding the deadlock.
         */
-        if (!down_read_trylock(&mm->mmap_sem)) {
+        if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
                if ((error_code & PF_USER) == 0 &&
-                    !search_exception_tables(regs->ip))
+                    !search_exception_tables(regs->ip)) {
-                        goto bad_area_nosemaphore;
+                        bad_area_nosemaphore(regs, error_code, address);
+                        return;
+                }
                down_read(&mm->mmap_sem);
        }
        vma = find_vma(mm, address);
-        if (!vma)
+        if (unlikely(!vma)) {
-                goto bad_area;
+                bad_area(regs, error_code, address);
-        if (vma->vm_start <= address)
+                return;
+        }
+        if (likely(vma->vm_start <= address))
                goto good_area;
-        if (!(vma->vm_flags & VM_GROWSDOWN))
+        if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
-                goto bad_area;
+                bad_area(regs, error_code, address);
+                return;
+        }
        if (error_code & PF_USER) {
                /*
                 * Accessing the stack below %sp is always a bug.
@@ -708,31 +919,25 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                 * and pusha to work.  ("enter $65535,$31" pushes
                 * 32 pointers and then decrements %sp by 65535.)
                 */
-                if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
+                if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
-                        goto bad_area;
+                        bad_area(regs, error_code, address);
+                        return;
+                }
        }
-        if (expand_stack(vma, address))
+        if (unlikely(expand_stack(vma, address))) {
-                goto bad_area;
+                bad_area(regs, error_code, address);
-/*
+                return;
- * Ok, we have a good vm_area for this memory access, so
+        }
- * we can handle it..
- */
+        /*
+         * Ok, we have a good vm_area for this memory access, so
+         * we can handle it..
+         */
 good_area:
-        si_code = SEGV_ACCERR;
+        write = error_code & PF_WRITE;
-        write = 0;
+        if (unlikely(access_error(error_code, write, vma))) {
-        switch (error_code & (PF_PROT|PF_WRITE)) {
+                bad_area_access_error(regs, error_code, address);
-        default:        /* 3: write, present */
+                return;
-                /* fall through */
-        case PF_WRITE:          /* write, not present */
-                if (!(vma->vm_flags & VM_WRITE))
-                        goto bad_area;
-                write++;
-                break;
-        case PF_PROT:           /* read, present */
-                goto bad_area;
-        case 0:                 /* read, not present */
-                if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
-                        goto bad_area;
        }
        /*
@@ -742,11 +947,8 @@ good_area:
         */
        fault = handle_mm_fault(mm, vma, address, write);
        if (unlikely(fault & VM_FAULT_ERROR)) {
-                if (fault & VM_FAULT_OOM)
+                mm_fault_error(regs, error_code, address, fault);
-                        goto out_of_memory;
+                return;
-                else if (fault & VM_FAULT_SIGBUS)
-                        goto do_sigbus;
-                BUG();
        }
        if (fault & VM_FAULT_MAJOR)
                tsk->maj_flt++;
@@ -764,128 +966,6 @@ good_area:
        }
 #endif
        up_read(&mm->mmap_sem);
-        return;
-/*
- * Something tried to access memory that isn't in our memory map..
- * Fix it, but check if it's kernel or user first..
- */
-bad_area:
-        up_read(&mm->mmap_sem);
-bad_area_nosemaphore:
-        /* User mode accesses just cause a SIGSEGV */
-        if (error_code & PF_USER) {
-                /*
-                 * It's possible to have interrupts off here.
-                 */
-                local_irq_enable();
-                /*
-                 * Valid to do another page fault here because this one came
-                 * from user space.
-                 */
-                if (is_prefetch(regs, address, error_code))
-                        return;
-                if (is_errata100(regs, address))
-                        return;
-                if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
-                    printk_ratelimit()) {
-                        printk(
-                        "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
-                        task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
-                        tsk->comm, task_pid_nr(tsk), address,
-                        (void *) regs->ip, (void *) regs->sp, error_code);
-                        print_vma_addr(" in ", regs->ip);
-                        printk("\n");
-                }
-                tsk->thread.cr2 = address;
-                /* Kernel addresses are always protection faults */
-                tsk->thread.error_code = error_code | (address >= TASK_SIZE);
-                tsk->thread.trap_no = 14;
-                force_sig_info_fault(SIGSEGV, si_code, address, tsk);
-                return;
-        }
-        if (is_f00f_bug(regs, address))
-                return;
-no_context:
-        /* Are we prepared to handle this kernel fault?  */
-        if (fixup_exception(regs))
-                return;
-        /*
-         * X86_32
-         * Valid to do another page fault here, because if this fault
-         * had been triggered by is_prefetch fixup_exception would have
-         * handled it.
-         *
-         * X86_64
-         * Hall of shame of CPU/BIOS bugs.
-         */
-        if (is_prefetch(regs, address, error_code))
-                return;
-        if (is_errata93(regs, address))
-                return;
-/*
- * Oops. The kernel tried to access some bad page. We'll have to
- * terminate things with extreme prejudice.
- */
-#ifdef CONFIG_X86_32
-        bust_spinlocks(1);
-#else
-        flags = oops_begin();
-#endif
-        show_fault_oops(regs, error_code, address);
-        tsk->thread.cr2 = address;
-        tsk->thread.trap_no = 14;
-        tsk->thread.error_code = error_code;
-#ifdef CONFIG_X86_32
-        die("Oops", regs, error_code);
-        bust_spinlocks(0);
-        do_exit(SIGKILL);
-#else
-        sig = SIGKILL;
-        if (__die("Oops", regs, error_code))
-                sig = 0;
-        /* Executive summary in case the body of the oops scrolled away */
-        printk(KERN_EMERG "CR2: %016lx\n", address);
-        oops_end(flags, regs, sig);
-#endif
-out_of_memory:
-        /*
-         * We ran out of memory, call the OOM killer, and return the userspace
-         * (which will retry the fault, or kill us if we got oom-killed).
-         */
-        up_read(&mm->mmap_sem);
-        pagefault_out_of_memory();
-        return;
-do_sigbus:
-        up_read(&mm->mmap_sem);
-        /* Kernel mode? Handle exceptions or die */
-        if (!(error_code & PF_USER))
-                goto no_context;
-#ifdef CONFIG_X86_32
-        /* User space => ok to do another page fault */
-        if (is_prefetch(regs, address, error_code))
-                return;
-#endif
-        tsk->thread.cr2 = address;
-        tsk->thread.error_code = error_code;
-        tsk->thread.trap_no = 14;
-        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
 }
 DEFINE_SPINLOCK(pgd_lock);

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index c76ef1d701c9..8c3f3113a6ec 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c
@@ -26,6 +26,7 @@
26	#include <linux/kprobes.h>	26	#include <linux/kprobes.h>
27	#include <linux/uaccess.h>	27	#include <linux/uaccess.h>
28	#include <linux/kdebug.h>	28	#include <linux/kdebug.h>
		29	#include <linux/magic.h>
29		30
30	#include <asm/system.h>	31	#include <asm/system.h>
31	#include <asm/desc.h>	32	#include <asm/desc.h>
@@ -91,8 +92,8 @@ static inline int notify_page_fault(struct pt_regs *regs)
91	*	92	*
92	* Opcode checker based on code by Richard Brunner	93	* Opcode checker based on code by Richard Brunner
93	*/	94	*/
94	static int is_prefetch(struct pt_regs *regs, unsigned long addr,	95	static int is_prefetch(struct pt_regs *regs, unsigned long error_code,
95	unsigned long error_code)	96	unsigned long addr)
96	{	97	{
97	unsigned char *instr;	98	unsigned char *instr;
98	int scan_more = 1;	99	int scan_more = 1;
@@ -409,15 +410,15 @@ static void show_fault_oops(struct pt_regs *regs, unsigned long error_code,
409	}	410	}
410		411
411	#ifdef CONFIG_X86_64	412	#ifdef CONFIG_X86_64
412	static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,	413	static noinline void pgtable_bad(struct pt_regs *regs,
413	unsigned long error_code)	414	unsigned long error_code, unsigned long address)
414	{	415	{
415	unsigned long flags = oops_begin();	416	unsigned long flags = oops_begin();
416	int sig = SIGKILL;	417	int sig = SIGKILL;
417	struct task_struct *tsk;	418	struct task_struct *tsk = current;
418		419
419	printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",	420	printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
420	current->comm, address);	421	tsk->comm, address);
421	dump_pagetable(address);	422	dump_pagetable(address);
422	tsk = current;	423	tsk = current;
423	tsk->thread.cr2 = address;	424	tsk->thread.cr2 = address;
@@ -429,6 +430,196 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
429	}	430	}
430	#endif	431	#endif
431		432
		433	static noinline void no_context(struct pt_regs *regs,
		434	unsigned long error_code, unsigned long address)
		435	{
		436	struct task_struct *tsk = current;
		437	unsigned long *stackend;
		438
		439	#ifdef CONFIG_X86_64
		440	unsigned long flags;
		441	int sig;
		442	#endif
		443
		444	/* Are we prepared to handle this kernel fault? */
		445	if (fixup_exception(regs))
		446	return;
		447
		448	/*
		449	* X86_32
		450	* Valid to do another page fault here, because if this fault
		451	* had been triggered by is_prefetch fixup_exception would have
		452	* handled it.
		453	*
		454	* X86_64
		455	* Hall of shame of CPU/BIOS bugs.
		456	*/
		457	if (is_prefetch(regs, error_code, address))
		458	return;
		459
		460	if (is_errata93(regs, address))
		461	return;
		462
		463	/*
		464	* Oops. The kernel tried to access some bad page. We'll have to
		465	* terminate things with extreme prejudice.
		466	*/
		467	#ifdef CONFIG_X86_32
		468	bust_spinlocks(1);
		469	#else
		470	flags = oops_begin();
		471	#endif
		472
		473	show_fault_oops(regs, error_code, address);
		474
		475	stackend = end_of_stack(tsk);
		476	if (*stackend != STACK_END_MAGIC)
		477	printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
		478
		479	tsk->thread.cr2 = address;
		480	tsk->thread.trap_no = 14;
		481	tsk->thread.error_code = error_code;
		482
		483	#ifdef CONFIG_X86_32
		484	die("Oops", regs, error_code);
		485	bust_spinlocks(0);
		486	do_exit(SIGKILL);
		487	#else
		488	sig = SIGKILL;
		489	if (__die("Oops", regs, error_code))
		490	sig = 0;
		491	/* Executive summary in case the body of the oops scrolled away */
		492	printk(KERN_EMERG "CR2: %016lx\n", address);
		493	oops_end(flags, regs, sig);
		494	#endif
		495	}
		496
		497	static void __bad_area_nosemaphore(struct pt_regs *regs,
		498	unsigned long error_code, unsigned long address,
		499	int si_code)
		500	{
		501	struct task_struct *tsk = current;
		502
		503	/* User mode accesses just cause a SIGSEGV */
		504	if (error_code & PF_USER) {
		505	/*
		506	* It's possible to have interrupts off here.
		507	*/
		508	local_irq_enable();
		509
		510	/*
		511	* Valid to do another page fault here because this one came
		512	* from user space.
		513	*/
		514	if (is_prefetch(regs, error_code, address))
		515	return;
		516
		517	if (is_errata100(regs, address))
		518	return;
		519
		520	if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
		521	printk_ratelimit()) {
		522	printk(
		523	"%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
		524	task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
		525	tsk->comm, task_pid_nr(tsk), address,
		526	(void ) regs->ip, (void ) regs->sp, error_code);
		527	print_vma_addr(" in ", regs->ip);
		528	printk("\n");
		529	}
		530
		531	tsk->thread.cr2 = address;
		532	/* Kernel addresses are always protection faults */
		533	tsk->thread.error_code = error_code \| (address >= TASK_SIZE);
		534	tsk->thread.trap_no = 14;
		535	force_sig_info_fault(SIGSEGV, si_code, address, tsk);
		536	return;
		537	}
		538
		539	if (is_f00f_bug(regs, address))
		540	return;
		541
		542	no_context(regs, error_code, address);
		543	}
		544
		545	static noinline void bad_area_nosemaphore(struct pt_regs *regs,
		546	unsigned long error_code, unsigned long address)
		547	{
		548	__bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
		549	}
		550
		551	static void __bad_area(struct pt_regs *regs,
		552	unsigned long error_code, unsigned long address,
		553	int si_code)
		554	{
		555	struct mm_struct *mm = current->mm;
		556
		557	/*
		558	* Something tried to access memory that isn't in our memory map..
		559	* Fix it, but check if it's kernel or user first..
		560	*/
		561	up_read(&mm->mmap_sem);
		562
		563	__bad_area_nosemaphore(regs, error_code, address, si_code);
		564	}
		565
		566	static noinline void bad_area(struct pt_regs *regs,
		567	unsigned long error_code, unsigned long address)
		568	{
		569	__bad_area(regs, error_code, address, SEGV_MAPERR);
		570	}
		571
		572	static noinline void bad_area_access_error(struct pt_regs *regs,
		573	unsigned long error_code, unsigned long address)
		574	{
		575	__bad_area(regs, error_code, address, SEGV_ACCERR);
		576	}
		577
		578	/* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */
		579	static void out_of_memory(struct pt_regs *regs,
		580	unsigned long error_code, unsigned long address)
		581	{
		582	/*
		583	* We ran out of memory, call the OOM killer, and return the userspace
		584	* (which will retry the fault, or kill us if we got oom-killed).
		585	*/
		586	up_read(&current->mm->mmap_sem);
		587	pagefault_out_of_memory();
		588	}
		589
		590	static void do_sigbus(struct pt_regs *regs,
		591	unsigned long error_code, unsigned long address)
		592	{
		593	struct task_struct *tsk = current;
		594	struct mm_struct *mm = tsk->mm;
		595
		596	up_read(&mm->mmap_sem);
		597
		598	/* Kernel mode? Handle exceptions or die */
		599	if (!(error_code & PF_USER))
		600	no_context(regs, error_code, address);
		601	#ifdef CONFIG_X86_32
		602	/* User space => ok to do another page fault */
		603	if (is_prefetch(regs, error_code, address))
		604	return;
		605	#endif
		606	tsk->thread.cr2 = address;
		607	tsk->thread.error_code = error_code;
		608	tsk->thread.trap_no = 14;
		609	force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
		610	}
		611
		612	static noinline void mm_fault_error(struct pt_regs *regs,
		613	unsigned long error_code, unsigned long address, unsigned int fault)
		614	{
		615	if (fault & VM_FAULT_OOM)
		616	out_of_memory(regs, error_code, address);
		617	else if (fault & VM_FAULT_SIGBUS)
		618	do_sigbus(regs, error_code, address);
		619	else
		620	BUG();
		621	}
		622
432	static int spurious_fault_check(unsigned long error_code, pte_t *pte)	623	static int spurious_fault_check(unsigned long error_code, pte_t *pte)
433	{	624	{
434	if ((error_code & PF_WRITE) && !pte_write(*pte))	625	if ((error_code & PF_WRITE) && !pte_write(*pte))
@@ -448,8 +639,8 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
448	* There are no security implications to leaving a stale TLB when	639	* There are no security implications to leaving a stale TLB when
449	* increasing the permissions on a page.	640	* increasing the permissions on a page.
450	*/	641	*/
451	static int spurious_fault(unsigned long address,	642	static noinline int spurious_fault(unsigned long error_code,
452	unsigned long error_code)	643	unsigned long address)
453	{	644	{
454	pgd_t *pgd;	645	pgd_t *pgd;
455	pud_t *pud;	646	pud_t *pud;
@@ -494,7 +685,7 @@ static int spurious_fault(unsigned long address,
494	*	685	*
495	* This assumes no large pages in there.	686	* This assumes no large pages in there.
496	*/	687	*/
497	static int vmalloc_fault(unsigned long address)	688	static noinline int vmalloc_fault(unsigned long address)
498	{	689	{
499	#ifdef CONFIG_X86_32	690	#ifdef CONFIG_X86_32
500	unsigned long pgd_paddr;	691	unsigned long pgd_paddr;
@@ -573,6 +764,25 @@ static int vmalloc_fault(unsigned long address)
573		764
574	int show_unhandled_signals = 1;	765	int show_unhandled_signals = 1;
575		766
		767	static inline int access_error(unsigned long error_code, int write,
		768	struct vm_area_struct *vma)
		769	{
		770	if (write) {
		771	/* write, present and write, not present */
		772	if (unlikely(!(vma->vm_flags & VM_WRITE)))
		773	return 1;
		774	} else if (unlikely(error_code & PF_PROT)) {
		775	/* read, present */
		776	return 1;
		777	} else {
		778	/* read, not present */
		779	if (unlikely(!(vma->vm_flags & (VM_READ \| VM_EXEC \| VM_WRITE))))
		780	return 1;
		781	}
		782
		783	return 0;
		784	}
		785
576	/*	786	/*
577	* This routine handles page faults. It determines the address,	787	* This routine handles page faults. It determines the address,
578	* and the problem, and then passes it off to one of the appropriate	788	* and the problem, and then passes it off to one of the appropriate
@@ -583,16 +793,12 @@ asmlinkage
583	#endif	793	#endif
584	void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)	794	void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
585	{	795	{
		796	unsigned long address;
586	struct task_struct *tsk;	797	struct task_struct *tsk;
587	struct mm_struct *mm;	798	struct mm_struct *mm;
588	struct vm_area_struct *vma;	799	struct vm_area_struct *vma;
589	unsigned long address;	800	int write;
590	int write, si_code;
591	int fault;	801	int fault;
592	#ifdef CONFIG_X86_64
593	unsigned long flags;
594	int sig;
595	#endif
596		802
597	tsk = current;	803	tsk = current;
598	mm = tsk->mm;	804	mm = tsk->mm;
@@ -601,8 +807,6 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
601	/* get the address */	807	/* get the address */
602	address = read_cr2();	808	address = read_cr2();
603		809
604	si_code = SEGV_MAPERR;
605
606	if (unlikely(kmmio_fault(regs, address)))	810	if (unlikely(kmmio_fault(regs, address)))
607	return;	811	return;
608		812
@@ -629,7 +833,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
629	return;	833	return;
630		834
631	/* Can handle a stale RO->RW TLB */	835	/* Can handle a stale RO->RW TLB */
632	if (spurious_fault(address, error_code))	836	if (spurious_fault(error_code, address))
633	return;	837	return;
634		838
635	/* kprobes don't want to hook the spurious faults. */	839	/* kprobes don't want to hook the spurious faults. */
@@ -639,13 +843,12 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
639	* Don't take the mm semaphore here. If we fixup a prefetch	843	* Don't take the mm semaphore here. If we fixup a prefetch
640	* fault we could otherwise deadlock.	844	* fault we could otherwise deadlock.
641	*/	845	*/
642	goto bad_area_nosemaphore;	846	bad_area_nosemaphore(regs, error_code, address);
		847	return;
643	}	848	}
644		849
645	/* kprobes don't want to hook the spurious faults. */	850	if (unlikely(notify_page_fault(regs)))
646	if (notify_page_fault(regs))
647	return;	851	return;
648
649	/*	852	/*
650	* It's safe to allow irq's after cr2 has been saved and the	853	* It's safe to allow irq's after cr2 has been saved and the
651	* vmalloc fault has been handled.	854	* vmalloc fault has been handled.
@@ -661,15 +864,17 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
661		864
662	#ifdef CONFIG_X86_64	865	#ifdef CONFIG_X86_64
663	if (unlikely(error_code & PF_RSVD))	866	if (unlikely(error_code & PF_RSVD))
664	pgtable_bad(address, regs, error_code);	867	pgtable_bad(regs, error_code, address);
665	#endif	868	#endif
666		869
667	/*	870	/*
668	* If we're in an interrupt, have no user context or are running in an	871	* If we're in an interrupt, have no user context or are running in an
669	* atomic region then we must not take the fault.	872	* atomic region then we must not take the fault.
670	*/	873	*/
671	if (unlikely(in_atomic() \|\| !mm))	874	if (unlikely(in_atomic() \|\| !mm)) {
672	goto bad_area_nosemaphore;	875	bad_area_nosemaphore(regs, error_code, address);
		876	return;
		877	}
673		878
674	/*	879	/*
675	* When running in the kernel we expect faults to occur only to	880	* When running in the kernel we expect faults to occur only to
@@ -687,20 +892,26 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
687	* source. If this is invalid we can skip the address space check,	892	* source. If this is invalid we can skip the address space check,
688	* thus avoiding the deadlock.	893	* thus avoiding the deadlock.
689	*/	894	*/
690	if (!down_read_trylock(&mm->mmap_sem)) {	895	if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
691	if ((error_code & PF_USER) == 0 &&	896	if ((error_code & PF_USER) == 0 &&
692	!search_exception_tables(regs->ip))	897	!search_exception_tables(regs->ip)) {
693	goto bad_area_nosemaphore;	898	bad_area_nosemaphore(regs, error_code, address);
		899	return;
		900	}
694	down_read(&mm->mmap_sem);	901	down_read(&mm->mmap_sem);
695	}	902	}
696		903
697	vma = find_vma(mm, address);	904	vma = find_vma(mm, address);
698	if (!vma)	905	if (unlikely(!vma)) {
699	goto bad_area;	906	bad_area(regs, error_code, address);
700	if (vma->vm_start <= address)	907	return;
		908	}
		909	if (likely(vma->vm_start <= address))
701	goto good_area;	910	goto good_area;
702	if (!(vma->vm_flags & VM_GROWSDOWN))	911	if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
703	goto bad_area;	912	bad_area(regs, error_code, address);
		913	return;
		914	}
704	if (error_code & PF_USER) {	915	if (error_code & PF_USER) {
705	/*	916	/*
706	* Accessing the stack below %sp is always a bug.	917	* Accessing the stack below %sp is always a bug.
@@ -708,31 +919,25 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
708	* and pusha to work. ("enter $65535,$31" pushes	919	* and pusha to work. ("enter $65535,$31" pushes
709	* 32 pointers and then decrements %sp by 65535.)	920	* 32 pointers and then decrements %sp by 65535.)
710	*/	921	*/
711	if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)	922	if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
712	goto bad_area;	923	bad_area(regs, error_code, address);
		924	return;
		925	}
713	}	926	}
714	if (expand_stack(vma, address))	927	if (unlikely(expand_stack(vma, address))) {
715	goto bad_area;	928	bad_area(regs, error_code, address);
716	/*	929	return;
717	* Ok, we have a good vm_area for this memory access, so	930	}
718	* we can handle it..	931
719	*/	932	/*
		933	* Ok, we have a good vm_area for this memory access, so
		934	* we can handle it..
		935	*/
720	good_area:	936	good_area:
721	si_code = SEGV_ACCERR;	937	write = error_code & PF_WRITE;
722	write = 0;	938	if (unlikely(access_error(error_code, write, vma))) {
723	switch (error_code & (PF_PROT\|PF_WRITE)) {	939	bad_area_access_error(regs, error_code, address);
724	default: /* 3: write, present */	940	return;
725	/* fall through */
726	case PF_WRITE: /* write, not present */
727	if (!(vma->vm_flags & VM_WRITE))
728	goto bad_area;
729	write++;
730	break;
731	case PF_PROT: /* read, present */
732	goto bad_area;
733	case 0: /* read, not present */
734	if (!(vma->vm_flags & (VM_READ \| VM_EXEC \| VM_WRITE)))
735	goto bad_area;
736	}	941	}
737		942
738	/*	943	/*
@@ -742,11 +947,8 @@ good_area:
742	*/	947	*/
743	fault = handle_mm_fault(mm, vma, address, write);	948	fault = handle_mm_fault(mm, vma, address, write);
744	if (unlikely(fault & VM_FAULT_ERROR)) {	949	if (unlikely(fault & VM_FAULT_ERROR)) {
745	if (fault & VM_FAULT_OOM)	950	mm_fault_error(regs, error_code, address, fault);
746	goto out_of_memory;	951	return;
747	else if (fault & VM_FAULT_SIGBUS)
748	goto do_sigbus;
749	BUG();
750	}	952	}
751	if (fault & VM_FAULT_MAJOR)	953	if (fault & VM_FAULT_MAJOR)
752	tsk->maj_flt++;	954	tsk->maj_flt++;
@@ -764,128 +966,6 @@ good_area:
764	}	966	}
765	#endif	967	#endif
766	up_read(&mm->mmap_sem);	968	up_read(&mm->mmap_sem);
767	return;
768
769	/*
770	* Something tried to access memory that isn't in our memory map..
771	* Fix it, but check if it's kernel or user first..
772	*/
773	bad_area:
774	up_read(&mm->mmap_sem);
775
776	bad_area_nosemaphore:
777	/* User mode accesses just cause a SIGSEGV */
778	if (error_code & PF_USER) {
779	/*
780	* It's possible to have interrupts off here.
781	*/
782	local_irq_enable();
783
784	/*
785	* Valid to do another page fault here because this one came
786	* from user space.
787	*/
788	if (is_prefetch(regs, address, error_code))
789	return;
790
791	if (is_errata100(regs, address))
792	return;
793
794	if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
795	printk_ratelimit()) {
796	printk(
797	"%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
798	task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
799	tsk->comm, task_pid_nr(tsk), address,
800	(void ) regs->ip, (void ) regs->sp, error_code);
801	print_vma_addr(" in ", regs->ip);
802	printk("\n");
803	}
804
805	tsk->thread.cr2 = address;
806	/* Kernel addresses are always protection faults */
807	tsk->thread.error_code = error_code \| (address >= TASK_SIZE);
808	tsk->thread.trap_no = 14;
809	force_sig_info_fault(SIGSEGV, si_code, address, tsk);
810	return;
811	}
812
813	if (is_f00f_bug(regs, address))
814	return;
815
816	no_context:
817	/* Are we prepared to handle this kernel fault? */
818	if (fixup_exception(regs))
819	return;
820
821	/*
822	* X86_32
823	* Valid to do another page fault here, because if this fault
824	* had been triggered by is_prefetch fixup_exception would have
825	* handled it.
826	*
827	* X86_64
828	* Hall of shame of CPU/BIOS bugs.
829	*/
830	if (is_prefetch(regs, address, error_code))
831	return;
832
833	if (is_errata93(regs, address))
834	return;
835
836	/*
837	* Oops. The kernel tried to access some bad page. We'll have to
838	* terminate things with extreme prejudice.
839	*/
840	#ifdef CONFIG_X86_32
841	bust_spinlocks(1);
842	#else
843	flags = oops_begin();
844	#endif
845
846	show_fault_oops(regs, error_code, address);
847
848	tsk->thread.cr2 = address;
849	tsk->thread.trap_no = 14;
850	tsk->thread.error_code = error_code;
851
852	#ifdef CONFIG_X86_32
853	die("Oops", regs, error_code);
854	bust_spinlocks(0);
855	do_exit(SIGKILL);
856	#else
857	sig = SIGKILL;
858	if (__die("Oops", regs, error_code))
859	sig = 0;
860	/* Executive summary in case the body of the oops scrolled away */
861	printk(KERN_EMERG "CR2: %016lx\n", address);
862	oops_end(flags, regs, sig);
863	#endif
864
865	out_of_memory:
866	/*
867	* We ran out of memory, call the OOM killer, and return the userspace
868	* (which will retry the fault, or kill us if we got oom-killed).
869	*/
870	up_read(&mm->mmap_sem);
871	pagefault_out_of_memory();
872	return;
873
874	do_sigbus:
875	up_read(&mm->mmap_sem);
876
877	/* Kernel mode? Handle exceptions or die */
878	if (!(error_code & PF_USER))
879	goto no_context;
880	#ifdef CONFIG_X86_32
881	/* User space => ok to do another page fault */
882	if (is_prefetch(regs, address, error_code))
883	return;
884	#endif
885	tsk->thread.cr2 = address;
886	tsk->thread.error_code = error_code;
887	tsk->thread.trap_no = 14;
888	force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
889	}	969	}
890		970
891	DEFINE_SPINLOCK(pgd_lock);	971	DEFINE_SPINLOCK(pgd_lock);