aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/mm/fault_64.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/mm/fault_64.c')
-rw-r--r--arch/x86/mm/fault_64.c623
1 files changed, 0 insertions, 623 deletions
diff --git a/arch/x86/mm/fault_64.c b/arch/x86/mm/fault_64.c
deleted file mode 100644
index 0e26230669ca..000000000000
--- a/arch/x86/mm/fault_64.c
+++ /dev/null
@@ -1,623 +0,0 @@
1/*
2 * linux/arch/x86-64/mm/fault.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
6 */
7
8#include <linux/signal.h>
9#include <linux/sched.h>
10#include <linux/kernel.h>
11#include <linux/errno.h>
12#include <linux/string.h>
13#include <linux/types.h>
14#include <linux/ptrace.h>
15#include <linux/mman.h>
16#include <linux/mm.h>
17#include <linux/smp.h>
18#include <linux/interrupt.h>
19#include <linux/init.h>
20#include <linux/tty.h>
21#include <linux/vt_kern.h> /* For unblank_screen() */
22#include <linux/compiler.h>
23#include <linux/vmalloc.h>
24#include <linux/module.h>
25#include <linux/kprobes.h>
26#include <linux/uaccess.h>
27#include <linux/kdebug.h>
28#include <linux/kprobes.h>
29
30#include <asm/system.h>
31#include <asm/pgalloc.h>
32#include <asm/smp.h>
33#include <asm/tlbflush.h>
34#include <asm/proto.h>
35#include <asm-generic/sections.h>
36
37/* Page fault error code bits */
38#define PF_PROT (1<<0) /* or no page found */
39#define PF_WRITE (1<<1)
40#define PF_USER (1<<2)
41#define PF_RSVD (1<<3)
42#define PF_INSTR (1<<4)
43
44#ifdef CONFIG_KPROBES
45static inline int notify_page_fault(struct pt_regs *regs)
46{
47 int ret = 0;
48
49 /* kprobe_running() needs smp_processor_id() */
50 if (!user_mode(regs)) {
51 preempt_disable();
52 if (kprobe_running() && kprobe_fault_handler(regs, 14))
53 ret = 1;
54 preempt_enable();
55 }
56
57 return ret;
58}
59#else
60static inline int notify_page_fault(struct pt_regs *regs)
61{
62 return 0;
63}
64#endif
65
66/* Sometimes the CPU reports invalid exceptions on prefetch.
67 Check that here and ignore.
68 Opcode checker based on code by Richard Brunner */
69static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
70 unsigned long error_code)
71{
72 unsigned char *instr;
73 int scan_more = 1;
74 int prefetch = 0;
75 unsigned char *max_instr;
76
77 /* If it was a exec fault ignore */
78 if (error_code & PF_INSTR)
79 return 0;
80
81 instr = (unsigned char __user *)convert_rip_to_linear(current, regs);
82 max_instr = instr + 15;
83
84 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
85 return 0;
86
87 while (scan_more && instr < max_instr) {
88 unsigned char opcode;
89 unsigned char instr_hi;
90 unsigned char instr_lo;
91
92 if (probe_kernel_address(instr, opcode))
93 break;
94
95 instr_hi = opcode & 0xf0;
96 instr_lo = opcode & 0x0f;
97 instr++;
98
99 switch (instr_hi) {
100 case 0x20:
101 case 0x30:
102 /* Values 0x26,0x2E,0x36,0x3E are valid x86
103 prefixes. In long mode, the CPU will signal
104 invalid opcode if some of these prefixes are
105 present so we will never get here anyway */
106 scan_more = ((instr_lo & 7) == 0x6);
107 break;
108
109 case 0x40:
110 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
111 Need to figure out under what instruction mode the
112 instruction was issued ... */
113 /* Could check the LDT for lm, but for now it's good
114 enough to assume that long mode only uses well known
115 segments or kernel. */
116 scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
117 break;
118
119 case 0x60:
120 /* 0x64 thru 0x67 are valid prefixes in all modes. */
121 scan_more = (instr_lo & 0xC) == 0x4;
122 break;
123 case 0xF0:
124 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
125 scan_more = !instr_lo || (instr_lo>>1) == 1;
126 break;
127 case 0x00:
128 /* Prefetch instruction is 0x0F0D or 0x0F18 */
129 scan_more = 0;
130 if (probe_kernel_address(instr, opcode))
131 break;
132 prefetch = (instr_lo == 0xF) &&
133 (opcode == 0x0D || opcode == 0x18);
134 break;
135 default:
136 scan_more = 0;
137 break;
138 }
139 }
140 return prefetch;
141}
142
143static int bad_address(void *p)
144{
145 unsigned long dummy;
146 return probe_kernel_address((unsigned long *)p, dummy);
147}
148
149void dump_pagetable(unsigned long address)
150{
151 pgd_t *pgd;
152 pud_t *pud;
153 pmd_t *pmd;
154 pte_t *pte;
155
156 pgd = (pgd_t *)read_cr3();
157
158 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
159 pgd += pgd_index(address);
160 if (bad_address(pgd)) goto bad;
161 printk("PGD %lx ", pgd_val(*pgd));
162 if (!pgd_present(*pgd)) goto ret;
163
164 pud = pud_offset(pgd, address);
165 if (bad_address(pud)) goto bad;
166 printk("PUD %lx ", pud_val(*pud));
167 if (!pud_present(*pud)) goto ret;
168
169 pmd = pmd_offset(pud, address);
170 if (bad_address(pmd)) goto bad;
171 printk("PMD %lx ", pmd_val(*pmd));
172 if (!pmd_present(*pmd) || pmd_large(*pmd)) goto ret;
173
174 pte = pte_offset_kernel(pmd, address);
175 if (bad_address(pte)) goto bad;
176 printk("PTE %lx", pte_val(*pte));
177ret:
178 printk("\n");
179 return;
180bad:
181 printk("BAD\n");
182}
183
184static const char errata93_warning[] =
185KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
186KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
187KERN_ERR "******* Please consider a BIOS update.\n"
188KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
189
190/* Workaround for K8 erratum #93 & buggy BIOS.
191 BIOS SMM functions are required to use a specific workaround
192 to avoid corruption of the 64bit RIP register on C stepping K8.
193 A lot of BIOS that didn't get tested properly miss this.
194 The OS sees this as a page fault with the upper 32bits of RIP cleared.
195 Try to work around it here.
196 Note we only handle faults in kernel here. */
197
198static int is_errata93(struct pt_regs *regs, unsigned long address)
199{
200 static int warned;
201 if (address != regs->rip)
202 return 0;
203 if ((address >> 32) != 0)
204 return 0;
205 address |= 0xffffffffUL << 32;
206 if ((address >= (u64)_stext && address <= (u64)_etext) ||
207 (address >= MODULES_VADDR && address <= MODULES_END)) {
208 if (!warned) {
209 printk(errata93_warning);
210 warned = 1;
211 }
212 regs->rip = address;
213 return 1;
214 }
215 return 0;
216}
217
218static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
219 unsigned long error_code)
220{
221 unsigned long flags = oops_begin();
222 struct task_struct *tsk;
223
224 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
225 current->comm, address);
226 dump_pagetable(address);
227 tsk = current;
228 tsk->thread.cr2 = address;
229 tsk->thread.trap_no = 14;
230 tsk->thread.error_code = error_code;
231 __die("Bad pagetable", regs, error_code);
232 oops_end(flags);
233 do_exit(SIGKILL);
234}
235
236/*
237 * Handle a fault on the vmalloc area
238 *
239 * This assumes no large pages in there.
240 */
241static int vmalloc_fault(unsigned long address)
242{
243 pgd_t *pgd, *pgd_ref;
244 pud_t *pud, *pud_ref;
245 pmd_t *pmd, *pmd_ref;
246 pte_t *pte, *pte_ref;
247
248 /* Copy kernel mappings over when needed. This can also
249 happen within a race in page table update. In the later
250 case just flush. */
251
252 pgd = pgd_offset(current->mm ?: &init_mm, address);
253 pgd_ref = pgd_offset_k(address);
254 if (pgd_none(*pgd_ref))
255 return -1;
256 if (pgd_none(*pgd))
257 set_pgd(pgd, *pgd_ref);
258 else
259 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
260
261 /* Below here mismatches are bugs because these lower tables
262 are shared */
263
264 pud = pud_offset(pgd, address);
265 pud_ref = pud_offset(pgd_ref, address);
266 if (pud_none(*pud_ref))
267 return -1;
268 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
269 BUG();
270 pmd = pmd_offset(pud, address);
271 pmd_ref = pmd_offset(pud_ref, address);
272 if (pmd_none(*pmd_ref))
273 return -1;
274 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
275 BUG();
276 pte_ref = pte_offset_kernel(pmd_ref, address);
277 if (!pte_present(*pte_ref))
278 return -1;
279 pte = pte_offset_kernel(pmd, address);
280 /* Don't use pte_page here, because the mappings can point
281 outside mem_map, and the NUMA hash lookup cannot handle
282 that. */
283 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
284 BUG();
285 return 0;
286}
287
288int show_unhandled_signals = 1;
289
290/*
291 * This routine handles page faults. It determines the address,
292 * and the problem, and then passes it off to one of the appropriate
293 * routines.
294 */
295asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
296 unsigned long error_code)
297{
298 struct task_struct *tsk;
299 struct mm_struct *mm;
300 struct vm_area_struct * vma;
301 unsigned long address;
302 const struct exception_table_entry *fixup;
303 int write, fault;
304 unsigned long flags;
305 siginfo_t info;
306
307 /*
308 * We can fault from pretty much anywhere, with unknown IRQ state.
309 */
310 trace_hardirqs_fixup();
311
312 tsk = current;
313 mm = tsk->mm;
314 prefetchw(&mm->mmap_sem);
315
316 /* get the address */
317 address = read_cr2();
318
319 info.si_code = SEGV_MAPERR;
320
321
322 /*
323 * We fault-in kernel-space virtual memory on-demand. The
324 * 'reference' page table is init_mm.pgd.
325 *
326 * NOTE! We MUST NOT take any locks for this case. We may
327 * be in an interrupt or a critical region, and should
328 * only copy the information from the master page table,
329 * nothing more.
330 *
331 * This verifies that the fault happens in kernel space
332 * (error_code & 4) == 0, and that the fault was not a
333 * protection error (error_code & 9) == 0.
334 */
335 if (unlikely(address >= TASK_SIZE64)) {
336 /*
337 * Don't check for the module range here: its PML4
338 * is always initialized because it's shared with the main
339 * kernel text. Only vmalloc may need PML4 syncups.
340 */
341 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
342 ((address >= VMALLOC_START && address < VMALLOC_END))) {
343 if (vmalloc_fault(address) >= 0)
344 return;
345 }
346 if (notify_page_fault(regs))
347 return;
348 /*
349 * Don't take the mm semaphore here. If we fixup a prefetch
350 * fault we could otherwise deadlock.
351 */
352 goto bad_area_nosemaphore;
353 }
354
355 if (notify_page_fault(regs))
356 return;
357
358 if (likely(regs->eflags & X86_EFLAGS_IF))
359 local_irq_enable();
360
361 if (unlikely(error_code & PF_RSVD))
362 pgtable_bad(address, regs, error_code);
363
364 /*
365 * If we're in an interrupt or have no user
366 * context, we must not take the fault..
367 */
368 if (unlikely(in_atomic() || !mm))
369 goto bad_area_nosemaphore;
370
371 /*
372 * User-mode registers count as a user access even for any
373 * potential system fault or CPU buglet.
374 */
375 if (user_mode_vm(regs))
376 error_code |= PF_USER;
377
378 again:
379 /* When running in the kernel we expect faults to occur only to
380 * addresses in user space. All other faults represent errors in the
381 * kernel and should generate an OOPS. Unfortunately, in the case of an
382 * erroneous fault occurring in a code path which already holds mmap_sem
383 * we will deadlock attempting to validate the fault against the
384 * address space. Luckily the kernel only validly references user
385 * space from well defined areas of code, which are listed in the
386 * exceptions table.
387 *
388 * As the vast majority of faults will be valid we will only perform
389 * the source reference check when there is a possibility of a deadlock.
390 * Attempt to lock the address space, if we cannot we then validate the
391 * source. If this is invalid we can skip the address space check,
392 * thus avoiding the deadlock.
393 */
394 if (!down_read_trylock(&mm->mmap_sem)) {
395 if ((error_code & PF_USER) == 0 &&
396 !search_exception_tables(regs->rip))
397 goto bad_area_nosemaphore;
398 down_read(&mm->mmap_sem);
399 }
400
401 vma = find_vma(mm, address);
402 if (!vma)
403 goto bad_area;
404 if (likely(vma->vm_start <= address))
405 goto good_area;
406 if (!(vma->vm_flags & VM_GROWSDOWN))
407 goto bad_area;
408 if (error_code & 4) {
409 /* Allow userspace just enough access below the stack pointer
410 * to let the 'enter' instruction work.
411 */
412 if (address + 65536 + 32 * sizeof(unsigned long) < regs->rsp)
413 goto bad_area;
414 }
415 if (expand_stack(vma, address))
416 goto bad_area;
417/*
418 * Ok, we have a good vm_area for this memory access, so
419 * we can handle it..
420 */
421good_area:
422 info.si_code = SEGV_ACCERR;
423 write = 0;
424 switch (error_code & (PF_PROT|PF_WRITE)) {
425 default: /* 3: write, present */
426 /* fall through */
427 case PF_WRITE: /* write, not present */
428 if (!(vma->vm_flags & VM_WRITE))
429 goto bad_area;
430 write++;
431 break;
432 case PF_PROT: /* read, present */
433 goto bad_area;
434 case 0: /* read, not present */
435 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
436 goto bad_area;
437 }
438
439 /*
440 * If for any reason at all we couldn't handle the fault,
441 * make sure we exit gracefully rather than endlessly redo
442 * the fault.
443 */
444 fault = handle_mm_fault(mm, vma, address, write);
445 if (unlikely(fault & VM_FAULT_ERROR)) {
446 if (fault & VM_FAULT_OOM)
447 goto out_of_memory;
448 else if (fault & VM_FAULT_SIGBUS)
449 goto do_sigbus;
450 BUG();
451 }
452 if (fault & VM_FAULT_MAJOR)
453 tsk->maj_flt++;
454 else
455 tsk->min_flt++;
456 up_read(&mm->mmap_sem);
457 return;
458
459/*
460 * Something tried to access memory that isn't in our memory map..
461 * Fix it, but check if it's kernel or user first..
462 */
463bad_area:
464 up_read(&mm->mmap_sem);
465
466bad_area_nosemaphore:
467 /* User mode accesses just cause a SIGSEGV */
468 if (error_code & PF_USER) {
469
470 /*
471 * It's possible to have interrupts off here.
472 */
473 local_irq_enable();
474
475 if (is_prefetch(regs, address, error_code))
476 return;
477
478 /* Work around K8 erratum #100 K8 in compat mode
479 occasionally jumps to illegal addresses >4GB. We
480 catch this here in the page fault handler because
481 these addresses are not reachable. Just detect this
482 case and return. Any code segment in LDT is
483 compatibility mode. */
484 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
485 (address >> 32))
486 return;
487
488 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
489 printk_ratelimit()) {
490 printk(
491 "%s%s[%d]: segfault at %lx rip %lx rsp %lx error %lx\n",
492 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
493 tsk->comm, tsk->pid, address, regs->rip,
494 regs->rsp, error_code);
495 }
496
497 tsk->thread.cr2 = address;
498 /* Kernel addresses are always protection faults */
499 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
500 tsk->thread.trap_no = 14;
501 info.si_signo = SIGSEGV;
502 info.si_errno = 0;
503 /* info.si_code has been set above */
504 info.si_addr = (void __user *)address;
505 force_sig_info(SIGSEGV, &info, tsk);
506 return;
507 }
508
509no_context:
510
511 /* Are we prepared to handle this kernel fault? */
512 fixup = search_exception_tables(regs->rip);
513 if (fixup) {
514 regs->rip = fixup->fixup;
515 return;
516 }
517
518 /*
519 * Hall of shame of CPU/BIOS bugs.
520 */
521
522 if (is_prefetch(regs, address, error_code))
523 return;
524
525 if (is_errata93(regs, address))
526 return;
527
528/*
529 * Oops. The kernel tried to access some bad page. We'll have to
530 * terminate things with extreme prejudice.
531 */
532
533 flags = oops_begin();
534
535 if (address < PAGE_SIZE)
536 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
537 else
538 printk(KERN_ALERT "Unable to handle kernel paging request");
539 printk(" at %016lx RIP: \n" KERN_ALERT,address);
540 printk_address(regs->rip);
541 dump_pagetable(address);
542 tsk->thread.cr2 = address;
543 tsk->thread.trap_no = 14;
544 tsk->thread.error_code = error_code;
545 __die("Oops", regs, error_code);
546 /* Executive summary in case the body of the oops scrolled away */
547 printk(KERN_EMERG "CR2: %016lx\n", address);
548 oops_end(flags);
549 do_exit(SIGKILL);
550
551/*
552 * We ran out of memory, or some other thing happened to us that made
553 * us unable to handle the page fault gracefully.
554 */
555out_of_memory:
556 up_read(&mm->mmap_sem);
557 if (is_global_init(current)) {
558 yield();
559 goto again;
560 }
561 printk("VM: killing process %s\n", tsk->comm);
562 if (error_code & 4)
563 do_group_exit(SIGKILL);
564 goto no_context;
565
566do_sigbus:
567 up_read(&mm->mmap_sem);
568
569 /* Kernel mode? Handle exceptions or die */
570 if (!(error_code & PF_USER))
571 goto no_context;
572
573 tsk->thread.cr2 = address;
574 tsk->thread.error_code = error_code;
575 tsk->thread.trap_no = 14;
576 info.si_signo = SIGBUS;
577 info.si_errno = 0;
578 info.si_code = BUS_ADRERR;
579 info.si_addr = (void __user *)address;
580 force_sig_info(SIGBUS, &info, tsk);
581 return;
582}
583
584DEFINE_SPINLOCK(pgd_lock);
585LIST_HEAD(pgd_list);
586
587void vmalloc_sync_all(void)
588{
589 /* Note that races in the updates of insync and start aren't
590 problematic:
591 insync can only get set bits added, and updates to start are only
592 improving performance (without affecting correctness if undone). */
593 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
594 static unsigned long start = VMALLOC_START & PGDIR_MASK;
595 unsigned long address;
596
597 for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
598 if (!test_bit(pgd_index(address), insync)) {
599 const pgd_t *pgd_ref = pgd_offset_k(address);
600 struct page *page;
601
602 if (pgd_none(*pgd_ref))
603 continue;
604 spin_lock(&pgd_lock);
605 list_for_each_entry(page, &pgd_list, lru) {
606 pgd_t *pgd;
607 pgd = (pgd_t *)page_address(page) + pgd_index(address);
608 if (pgd_none(*pgd))
609 set_pgd(pgd, *pgd_ref);
610 else
611 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
612 }
613 spin_unlock(&pgd_lock);
614 set_bit(pgd_index(address), insync);
615 }
616 if (address == start)
617 start = address + PGDIR_SIZE;
618 }
619 /* Check that there is no need to do the same for the modules area. */
620 BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
621 BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
622 (__START_KERNEL & PGDIR_MASK)));
623}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-25 19:03:19 -0500