aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86_64/mm/fault.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@ppc970.osdl.org>2005-04-16 18:20:36 -0400
committerLinus Torvalds <torvalds@ppc970.osdl.org>2005-04-16 18:20:36 -0400
commit1da177e4c3f41524e886b7f1b8a0c1fc7321cac2 (patch)
tree0bba044c4ce775e45a88a51686b5d9f90697ea9d /arch/x86_64/mm/fault.c
Linux-2.6.12-rc2v2.6.12-rc2
Initial git repository build. I'm not bothering with the full history, even though we have it. We can create a separate "historical" git archive of that later if we want to, and in the meantime it's about 3.2GB when imported into git - space that would just make the early git days unnecessarily complicated, when we don't have a lot of good infrastructure for it. Let it rip!
Diffstat (limited to 'arch/x86_64/mm/fault.c')
-rw-r--r--arch/x86_64/mm/fault.c579
1 files changed, 579 insertions, 0 deletions
diff --git a/arch/x86_64/mm/fault.c b/arch/x86_64/mm/fault.c
new file mode 100644
index 000000000000..5724370475cc
--- /dev/null
+++ b/arch/x86_64/mm/fault.c
@@ -0,0 +1,579 @@
1/*
2 * linux/arch/x86-64/mm/fault.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
6 */
7
8#include <linux/config.h>
9#include <linux/signal.h>
10#include <linux/sched.h>
11#include <linux/kernel.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/types.h>
15#include <linux/ptrace.h>
16#include <linux/mman.h>
17#include <linux/mm.h>
18#include <linux/smp.h>
19#include <linux/smp_lock.h>
20#include <linux/interrupt.h>
21#include <linux/init.h>
22#include <linux/tty.h>
23#include <linux/vt_kern.h> /* For unblank_screen() */
24#include <linux/compiler.h>
25#include <linux/module.h>
26#include <linux/kprobes.h>
27
28#include <asm/system.h>
29#include <asm/uaccess.h>
30#include <asm/pgalloc.h>
31#include <asm/smp.h>
32#include <asm/tlbflush.h>
33#include <asm/proto.h>
34#include <asm/kdebug.h>
35#include <asm-generic/sections.h>
36#include <asm/kdebug.h>
37
38void bust_spinlocks(int yes)
39{
40 int loglevel_save = console_loglevel;
41 if (yes) {
42 oops_in_progress = 1;
43 } else {
44#ifdef CONFIG_VT
45 unblank_screen();
46#endif
47 oops_in_progress = 0;
48 /*
49 * OK, the message is on the console. Now we call printk()
50 * without oops_in_progress set so that printk will give klogd
51 * a poke. Hold onto your hats...
52 */
53 console_loglevel = 15; /* NMI oopser may have shut the console up */
54 printk(" ");
55 console_loglevel = loglevel_save;
56 }
57}
58
59/* Sometimes the CPU reports invalid exceptions on prefetch.
60 Check that here and ignore.
61 Opcode checker based on code by Richard Brunner */
62static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
63 unsigned long error_code)
64{
65 unsigned char *instr = (unsigned char *)(regs->rip);
66 int scan_more = 1;
67 int prefetch = 0;
68 unsigned char *max_instr = instr + 15;
69
70 /* If it was a exec fault ignore */
71 if (error_code & (1<<4))
72 return 0;
73
74 /* Code segments in LDT could have a non zero base. Don't check
75 when that's possible */
76 if (regs->cs & (1<<2))
77 return 0;
78
79 if ((regs->cs & 3) != 0 && regs->rip >= TASK_SIZE)
80 return 0;
81
82 while (scan_more && instr < max_instr) {
83 unsigned char opcode;
84 unsigned char instr_hi;
85 unsigned char instr_lo;
86
87 if (__get_user(opcode, instr))
88 break;
89
90 instr_hi = opcode & 0xf0;
91 instr_lo = opcode & 0x0f;
92 instr++;
93
94 switch (instr_hi) {
95 case 0x20:
96 case 0x30:
97 /* Values 0x26,0x2E,0x36,0x3E are valid x86
98 prefixes. In long mode, the CPU will signal
99 invalid opcode if some of these prefixes are
100 present so we will never get here anyway */
101 scan_more = ((instr_lo & 7) == 0x6);
102 break;
103
104 case 0x40:
105 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
106 Need to figure out under what instruction mode the
107 instruction was issued ... */
108 /* Could check the LDT for lm, but for now it's good
109 enough to assume that long mode only uses well known
110 segments or kernel. */
111 scan_more = ((regs->cs & 3) == 0) || (regs->cs == __USER_CS);
112 break;
113
114 case 0x60:
115 /* 0x64 thru 0x67 are valid prefixes in all modes. */
116 scan_more = (instr_lo & 0xC) == 0x4;
117 break;
118 case 0xF0:
119 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
120 scan_more = !instr_lo || (instr_lo>>1) == 1;
121 break;
122 case 0x00:
123 /* Prefetch instruction is 0x0F0D or 0x0F18 */
124 scan_more = 0;
125 if (__get_user(opcode, instr))
126 break;
127 prefetch = (instr_lo == 0xF) &&
128 (opcode == 0x0D || opcode == 0x18);
129 break;
130 default:
131 scan_more = 0;
132 break;
133 }
134 }
135 return prefetch;
136}
137
138static int bad_address(void *p)
139{
140 unsigned long dummy;
141 return __get_user(dummy, (unsigned long *)p);
142}
143
144void dump_pagetable(unsigned long address)
145{
146 pgd_t *pgd;
147 pud_t *pud;
148 pmd_t *pmd;
149 pte_t *pte;
150
151 asm("movq %%cr3,%0" : "=r" (pgd));
152
153 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
154 pgd += pgd_index(address);
155 printk("PGD %lx ", pgd_val(*pgd));
156 if (bad_address(pgd)) goto bad;
157 if (!pgd_present(*pgd)) goto ret;
158
159 pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
160 if (bad_address(pud)) goto bad;
161 printk("PUD %lx ", pud_val(*pud));
162 if (!pud_present(*pud)) goto ret;
163
164 pmd = pmd_offset(pud, address);
165 if (bad_address(pmd)) goto bad;
166 printk("PMD %lx ", pmd_val(*pmd));
167 if (!pmd_present(*pmd)) goto ret;
168
169 pte = pte_offset_kernel(pmd, address);
170 if (bad_address(pte)) goto bad;
171 printk("PTE %lx", pte_val(*pte));
172ret:
173 printk("\n");
174 return;
175bad:
176 printk("BAD\n");
177}
178
179static const char errata93_warning[] =
180KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
181KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
182KERN_ERR "******* Please consider a BIOS update.\n"
183KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
184
185/* Workaround for K8 erratum #93 & buggy BIOS.
186 BIOS SMM functions are required to use a specific workaround
187 to avoid corruption of the 64bit RIP register on C stepping K8.
188 A lot of BIOS that didn't get tested properly miss this.
189 The OS sees this as a page fault with the upper 32bits of RIP cleared.
190 Try to work around it here.
191 Note we only handle faults in kernel here. */
192
193static int is_errata93(struct pt_regs *regs, unsigned long address)
194{
195 static int warned;
196 if (address != regs->rip)
197 return 0;
198 if ((address >> 32) != 0)
199 return 0;
200 address |= 0xffffffffUL << 32;
201 if ((address >= (u64)_stext && address <= (u64)_etext) ||
202 (address >= MODULES_VADDR && address <= MODULES_END)) {
203 if (!warned) {
204 printk(errata93_warning);
205 warned = 1;
206 }
207 regs->rip = address;
208 return 1;
209 }
210 return 0;
211}
212
213int unhandled_signal(struct task_struct *tsk, int sig)
214{
215 if (tsk->pid == 1)
216 return 1;
217 /* Warn for strace, but not for gdb */
218 if (!test_ti_thread_flag(tsk->thread_info, TIF_SYSCALL_TRACE) &&
219 (tsk->ptrace & PT_PTRACED))
220 return 0;
221 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
222 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
223}
224
225static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
226 unsigned long error_code)
227{
228 oops_begin();
229 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
230 current->comm, address);
231 dump_pagetable(address);
232 __die("Bad pagetable", regs, error_code);
233 oops_end();
234 do_exit(SIGKILL);
235}
236
237/*
238 * Handle a fault on the vmalloc or module mapping area
239 */
240static int vmalloc_fault(unsigned long address)
241{
242 pgd_t *pgd, *pgd_ref;
243 pud_t *pud, *pud_ref;
244 pmd_t *pmd, *pmd_ref;
245 pte_t *pte, *pte_ref;
246
247 /* Copy kernel mappings over when needed. This can also
248 happen within a race in page table update. In the later
249 case just flush. */
250
251 pgd = pgd_offset(current->mm ?: &init_mm, address);
252 pgd_ref = pgd_offset_k(address);
253 if (pgd_none(*pgd_ref))
254 return -1;
255 if (pgd_none(*pgd))
256 set_pgd(pgd, *pgd_ref);
257
258 /* Below here mismatches are bugs because these lower tables
259 are shared */
260
261 pud = pud_offset(pgd, address);
262 pud_ref = pud_offset(pgd_ref, address);
263 if (pud_none(*pud_ref))
264 return -1;
265 if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
266 BUG();
267 pmd = pmd_offset(pud, address);
268 pmd_ref = pmd_offset(pud_ref, address);
269 if (pmd_none(*pmd_ref))
270 return -1;
271 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
272 BUG();
273 pte_ref = pte_offset_kernel(pmd_ref, address);
274 if (!pte_present(*pte_ref))
275 return -1;
276 pte = pte_offset_kernel(pmd, address);
277 if (!pte_present(*pte) || pte_page(*pte) != pte_page(*pte_ref))
278 BUG();
279 __flush_tlb_all();
280 return 0;
281}
282
283int page_fault_trace = 0;
284int exception_trace = 1;
285
286/*
287 * This routine handles page faults. It determines the address,
288 * and the problem, and then passes it off to one of the appropriate
289 * routines.
290 *
291 * error_code:
292 * bit 0 == 0 means no page found, 1 means protection fault
293 * bit 1 == 0 means read, 1 means write
294 * bit 2 == 0 means kernel, 1 means user-mode
295 * bit 3 == 1 means fault was an instruction fetch
296 */
297asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code)
298{
299 struct task_struct *tsk;
300 struct mm_struct *mm;
301 struct vm_area_struct * vma;
302 unsigned long address;
303 const struct exception_table_entry *fixup;
304 int write;
305 siginfo_t info;
306
307#ifdef CONFIG_CHECKING
308 {
309 unsigned long gs;
310 struct x8664_pda *pda = cpu_pda + stack_smp_processor_id();
311 rdmsrl(MSR_GS_BASE, gs);
312 if (gs != (unsigned long)pda) {
313 wrmsrl(MSR_GS_BASE, pda);
314 printk("page_fault: wrong gs %lx expected %p\n", gs, pda);
315 }
316 }
317#endif
318
319 /* get the address */
320 __asm__("movq %%cr2,%0":"=r" (address));
321 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
322 SIGSEGV) == NOTIFY_STOP)
323 return;
324
325 if (likely(regs->eflags & X86_EFLAGS_IF))
326 local_irq_enable();
327
328 if (unlikely(page_fault_trace))
329 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
330 regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
331
332 tsk = current;
333 mm = tsk->mm;
334 info.si_code = SEGV_MAPERR;
335
336
337 /*
338 * We fault-in kernel-space virtual memory on-demand. The
339 * 'reference' page table is init_mm.pgd.
340 *
341 * NOTE! We MUST NOT take any locks for this case. We may
342 * be in an interrupt or a critical region, and should
343 * only copy the information from the master page table,
344 * nothing more.
345 *
346 * This verifies that the fault happens in kernel space
347 * (error_code & 4) == 0, and that the fault was not a
348 * protection error (error_code & 1) == 0.
349 */
350 if (unlikely(address >= TASK_SIZE)) {
351 if (!(error_code & 5)) {
352 if (vmalloc_fault(address) < 0)
353 goto bad_area_nosemaphore;
354 return;
355 }
356 /*
357 * Don't take the mm semaphore here. If we fixup a prefetch
358 * fault we could otherwise deadlock.
359 */
360 goto bad_area_nosemaphore;
361 }
362
363 if (unlikely(error_code & (1 << 3)))
364 pgtable_bad(address, regs, error_code);
365
366 /*
367 * If we're in an interrupt or have no user
368 * context, we must not take the fault..
369 */
370 if (unlikely(in_atomic() || !mm))
371 goto bad_area_nosemaphore;
372
373 again:
374 /* When running in the kernel we expect faults to occur only to
375 * addresses in user space. All other faults represent errors in the
376 * kernel and should generate an OOPS. Unfortunatly, in the case of an
377 * erroneous fault occuring in a code path which already holds mmap_sem
378 * we will deadlock attempting to validate the fault against the
379 * address space. Luckily the kernel only validly references user
380 * space from well defined areas of code, which are listed in the
381 * exceptions table.
382 *
383 * As the vast majority of faults will be valid we will only perform
384 * the source reference check when there is a possibilty of a deadlock.
385 * Attempt to lock the address space, if we cannot we then validate the
386 * source. If this is invalid we can skip the address space check,
387 * thus avoiding the deadlock.
388 */
389 if (!down_read_trylock(&mm->mmap_sem)) {
390 if ((error_code & 4) == 0 &&
391 !search_exception_tables(regs->rip))
392 goto bad_area_nosemaphore;
393 down_read(&mm->mmap_sem);
394 }
395
396 vma = find_vma(mm, address);
397 if (!vma)
398 goto bad_area;
399 if (likely(vma->vm_start <= address))
400 goto good_area;
401 if (!(vma->vm_flags & VM_GROWSDOWN))
402 goto bad_area;
403 if (error_code & 4) {
404 // XXX: align red zone size with ABI
405 if (address + 128 < regs->rsp)
406 goto bad_area;
407 }
408 if (expand_stack(vma, address))
409 goto bad_area;
410/*
411 * Ok, we have a good vm_area for this memory access, so
412 * we can handle it..
413 */
414good_area:
415 info.si_code = SEGV_ACCERR;
416 write = 0;
417 switch (error_code & 3) {
418 default: /* 3: write, present */
419 /* fall through */
420 case 2: /* write, not present */
421 if (!(vma->vm_flags & VM_WRITE))
422 goto bad_area;
423 write++;
424 break;
425 case 1: /* read, present */
426 goto bad_area;
427 case 0: /* read, not present */
428 if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
429 goto bad_area;
430 }
431
432 /*
433 * If for any reason at all we couldn't handle the fault,
434 * make sure we exit gracefully rather than endlessly redo
435 * the fault.
436 */
437 switch (handle_mm_fault(mm, vma, address, write)) {
438 case 1:
439 tsk->min_flt++;
440 break;
441 case 2:
442 tsk->maj_flt++;
443 break;
444 case 0:
445 goto do_sigbus;
446 default:
447 goto out_of_memory;
448 }
449
450 up_read(&mm->mmap_sem);
451 return;
452
453/*
454 * Something tried to access memory that isn't in our memory map..
455 * Fix it, but check if it's kernel or user first..
456 */
457bad_area:
458 up_read(&mm->mmap_sem);
459
460bad_area_nosemaphore:
461
462#ifdef CONFIG_IA32_EMULATION
463 /* 32bit vsyscall. map on demand. */
464 if (test_thread_flag(TIF_IA32) &&
465 address >= VSYSCALL32_BASE && address < VSYSCALL32_END) {
466 if (map_syscall32(mm, address) < 0)
467 goto out_of_memory2;
468 return;
469 }
470#endif
471
472 /* User mode accesses just cause a SIGSEGV */
473 if (error_code & 4) {
474 if (is_prefetch(regs, address, error_code))
475 return;
476
477 /* Work around K8 erratum #100 K8 in compat mode
478 occasionally jumps to illegal addresses >4GB. We
479 catch this here in the page fault handler because
480 these addresses are not reachable. Just detect this
481 case and return. Any code segment in LDT is
482 compatibility mode. */
483 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
484 (address >> 32))
485 return;
486
487 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
488 printk(
489 "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
490 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
491 tsk->comm, tsk->pid, address, regs->rip,
492 regs->rsp, error_code);
493 }
494
495 tsk->thread.cr2 = address;
496 /* Kernel addresses are always protection faults */
497 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
498 tsk->thread.trap_no = 14;
499 info.si_signo = SIGSEGV;
500 info.si_errno = 0;
501 /* info.si_code has been set above */
502 info.si_addr = (void __user *)address;
503 force_sig_info(SIGSEGV, &info, tsk);
504 return;
505 }
506
507no_context:
508
509 /* Are we prepared to handle this kernel fault? */
510 fixup = search_exception_tables(regs->rip);
511 if (fixup) {
512 regs->rip = fixup->fixup;
513 return;
514 }
515
516 /*
517 * Hall of shame of CPU/BIOS bugs.
518 */
519
520 if (is_prefetch(regs, address, error_code))
521 return;
522
523 if (is_errata93(regs, address))
524 return;
525
526/*
527 * Oops. The kernel tried to access some bad page. We'll have to
528 * terminate things with extreme prejudice.
529 */
530
531 oops_begin();
532
533 if (address < PAGE_SIZE)
534 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
535 else
536 printk(KERN_ALERT "Unable to handle kernel paging request");
537 printk(" at %016lx RIP: \n" KERN_ALERT,address);
538 printk_address(regs->rip);
539 printk("\n");
540 dump_pagetable(address);
541 __die("Oops", regs, error_code);
542 /* Executive summary in case the body of the oops scrolled away */
543 printk(KERN_EMERG "CR2: %016lx\n", address);
544 oops_end();
545 do_exit(SIGKILL);
546
547/*
548 * We ran out of memory, or some other thing happened to us that made
549 * us unable to handle the page fault gracefully.
550 */
551out_of_memory:
552 up_read(&mm->mmap_sem);
553out_of_memory2:
554 if (current->pid == 1) {
555 yield();
556 goto again;
557 }
558 printk("VM: killing process %s\n", tsk->comm);
559 if (error_code & 4)
560 do_exit(SIGKILL);
561 goto no_context;
562
563do_sigbus:
564 up_read(&mm->mmap_sem);
565
566 /* Kernel mode? Handle exceptions or die */
567 if (!(error_code & 4))
568 goto no_context;
569
570 tsk->thread.cr2 = address;
571 tsk->thread.error_code = error_code;
572 tsk->thread.trap_no = 14;
573 info.si_signo = SIGBUS;
574 info.si_errno = 0;
575 info.si_code = BUS_ADRERR;
576 info.si_addr = (void __user *)address;
577 force_sig_info(SIGBUS, &info, tsk);
578 return;
579}