aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm/mm/fault.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm/fault.c')
-rw-r--r--arch/arm/mm/fault.c169
1 files changed, 58 insertions, 111 deletions
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index aa33949fef60..bb7eac381a8e 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -27,19 +27,6 @@
27 27
28#include "fault.h" 28#include "fault.h"
29 29
30/*
31 * Fault status register encodings. We steal bit 31 for our own purposes.
32 */
33#define FSR_LNX_PF (1 << 31)
34#define FSR_WRITE (1 << 11)
35#define FSR_FS4 (1 << 10)
36#define FSR_FS3_0 (15)
37
38static inline int fsr_fs(unsigned int fsr)
39{
40 return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
41}
42
43#ifdef CONFIG_MMU 30#ifdef CONFIG_MMU
44 31
45#ifdef CONFIG_KPROBES 32#ifdef CONFIG_KPROBES
@@ -123,8 +110,10 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
123 110
124 pte = pte_offset_map(pmd, addr); 111 pte = pte_offset_map(pmd, addr);
125 printk(", *pte=%08llx", (long long)pte_val(*pte)); 112 printk(", *pte=%08llx", (long long)pte_val(*pte));
113#ifndef CONFIG_ARM_LPAE
126 printk(", *ppte=%08llx", 114 printk(", *ppte=%08llx",
127 (long long)pte_val(pte[PTE_HWTABLE_PTRS])); 115 (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
116#endif
128 pte_unmap(pte); 117 pte_unmap(pte);
129 } while(0); 118 } while(0);
130 119
@@ -231,7 +220,7 @@ static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
231 220
232static int __kprobes 221static int __kprobes
233__do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, 222__do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
234 struct task_struct *tsk) 223 unsigned int flags, struct task_struct *tsk)
235{ 224{
236 struct vm_area_struct *vma; 225 struct vm_area_struct *vma;
237 int fault; 226 int fault;
@@ -253,18 +242,7 @@ good_area:
253 goto out; 242 goto out;
254 } 243 }
255 244
256 /* 245 return handle_mm_fault(mm, vma, addr & PAGE_MASK, flags);
257 * If for any reason at all we couldn't handle the fault, make
258 * sure we exit gracefully rather than endlessly redo the fault.
259 */
260 fault = handle_mm_fault(mm, vma, addr & PAGE_MASK, (fsr & FSR_WRITE) ? FAULT_FLAG_WRITE : 0);
261 if (unlikely(fault & VM_FAULT_ERROR))
262 return fault;
263 if (fault & VM_FAULT_MAJOR)
264 tsk->maj_flt++;
265 else
266 tsk->min_flt++;
267 return fault;
268 246
269check_stack: 247check_stack:
270 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr)) 248 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
@@ -279,6 +257,9 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
279 struct task_struct *tsk; 257 struct task_struct *tsk;
280 struct mm_struct *mm; 258 struct mm_struct *mm;
281 int fault, sig, code; 259 int fault, sig, code;
260 int write = fsr & FSR_WRITE;
261 unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
262 (write ? FAULT_FLAG_WRITE : 0);
282 263
283 if (notify_page_fault(regs, fsr)) 264 if (notify_page_fault(regs, fsr))
284 return 0; 265 return 0;
@@ -305,6 +286,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
305 if (!down_read_trylock(&mm->mmap_sem)) { 286 if (!down_read_trylock(&mm->mmap_sem)) {
306 if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc)) 287 if (!user_mode(regs) && !search_exception_tables(regs->ARM_pc))
307 goto no_context; 288 goto no_context;
289retry:
308 down_read(&mm->mmap_sem); 290 down_read(&mm->mmap_sem);
309 } else { 291 } else {
310 /* 292 /*
@@ -320,14 +302,41 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
320#endif 302#endif
321 } 303 }
322 304
323 fault = __do_page_fault(mm, addr, fsr, tsk); 305 fault = __do_page_fault(mm, addr, fsr, flags, tsk);
324 up_read(&mm->mmap_sem); 306
307 /* If we need to retry but a fatal signal is pending, handle the
308 * signal first. We do not need to release the mmap_sem because
309 * it would already be released in __lock_page_or_retry in
310 * mm/filemap.c. */
311 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
312 return 0;
313
314 /*
315 * Major/minor page fault accounting is only done on the
316 * initial attempt. If we go through a retry, it is extremely
317 * likely that the page will be found in page cache at that point.
318 */
325 319
326 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); 320 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
327 if (fault & VM_FAULT_MAJOR) 321 if (flags & FAULT_FLAG_ALLOW_RETRY) {
328 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, addr); 322 if (fault & VM_FAULT_MAJOR) {
329 else if (fault & VM_FAULT_MINOR) 323 tsk->maj_flt++;
330 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, addr); 324 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
325 regs, addr);
326 } else {
327 tsk->min_flt++;
328 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
329 regs, addr);
330 }
331 if (fault & VM_FAULT_RETRY) {
332 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
333 * of starvation. */
334 flags &= ~FAULT_FLAG_ALLOW_RETRY;
335 goto retry;
336 }
337 }
338
339 up_read(&mm->mmap_sem);
331 340
332 /* 341 /*
333 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR 342 * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
@@ -441,6 +450,12 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
441 pmd = pmd_offset(pud, addr); 450 pmd = pmd_offset(pud, addr);
442 pmd_k = pmd_offset(pud_k, addr); 451 pmd_k = pmd_offset(pud_k, addr);
443 452
453#ifdef CONFIG_ARM_LPAE
454 /*
455 * Only one hardware entry per PMD with LPAE.
456 */
457 index = 0;
458#else
444 /* 459 /*
445 * On ARM one Linux PGD entry contains two hardware entries (see page 460 * On ARM one Linux PGD entry contains two hardware entries (see page
446 * tables layout in pgtable.h). We normally guarantee that we always 461 * tables layout in pgtable.h). We normally guarantee that we always
@@ -450,6 +465,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
450 * for the first of pair. 465 * for the first of pair.
451 */ 466 */
452 index = (addr >> SECTION_SHIFT) & 1; 467 index = (addr >> SECTION_SHIFT) & 1;
468#endif
453 if (pmd_none(pmd_k[index])) 469 if (pmd_none(pmd_k[index]))
454 goto bad_area; 470 goto bad_area;
455 471
@@ -489,55 +505,20 @@ do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
489 return 1; 505 return 1;
490} 506}
491 507
492static struct fsr_info { 508struct fsr_info {
493 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs); 509 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
494 int sig; 510 int sig;
495 int code; 511 int code;
496 const char *name; 512 const char *name;
497} fsr_info[] = {
498 /*
499 * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
500 * defines these to be "precise" aborts.
501 */
502 { do_bad, SIGSEGV, 0, "vector exception" },
503 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
504 { do_bad, SIGKILL, 0, "terminal exception" },
505 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
506 { do_bad, SIGBUS, 0, "external abort on linefetch" },
507 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
508 { do_bad, SIGBUS, 0, "external abort on linefetch" },
509 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
510 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
511 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
512 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
513 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
514 { do_bad, SIGBUS, 0, "external abort on translation" },
515 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
516 { do_bad, SIGBUS, 0, "external abort on translation" },
517 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
518 /*
519 * The following are "imprecise" aborts, which are signalled by bit
520 * 10 of the FSR, and may not be recoverable. These are only
521 * supported if the CPU abort handler supports bit 10.
522 */
523 { do_bad, SIGBUS, 0, "unknown 16" },
524 { do_bad, SIGBUS, 0, "unknown 17" },
525 { do_bad, SIGBUS, 0, "unknown 18" },
526 { do_bad, SIGBUS, 0, "unknown 19" },
527 { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */
528 { do_bad, SIGBUS, 0, "unknown 21" },
529 { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */
530 { do_bad, SIGBUS, 0, "unknown 23" },
531 { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */
532 { do_bad, SIGBUS, 0, "unknown 25" },
533 { do_bad, SIGBUS, 0, "unknown 26" },
534 { do_bad, SIGBUS, 0, "unknown 27" },
535 { do_bad, SIGBUS, 0, "unknown 28" },
536 { do_bad, SIGBUS, 0, "unknown 29" },
537 { do_bad, SIGBUS, 0, "unknown 30" },
538 { do_bad, SIGBUS, 0, "unknown 31" }
539}; 513};
540 514
515/* FSR definition */
516#ifdef CONFIG_ARM_LPAE
517#include "fsr-3level.c"
518#else
519#include "fsr-2level.c"
520#endif
521
541void __init 522void __init
542hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), 523hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
543 int sig, int code, const char *name) 524 int sig, int code, const char *name)
@@ -573,42 +554,6 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
573 arm_notify_die("", regs, &info, fsr, 0); 554 arm_notify_die("", regs, &info, fsr, 0);
574} 555}
575 556
576
577static struct fsr_info ifsr_info[] = {
578 { do_bad, SIGBUS, 0, "unknown 0" },
579 { do_bad, SIGBUS, 0, "unknown 1" },
580 { do_bad, SIGBUS, 0, "debug event" },
581 { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" },
582 { do_bad, SIGBUS, 0, "unknown 4" },
583 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
584 { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" },
585 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
586 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
587 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
588 { do_bad, SIGBUS, 0, "unknown 10" },
589 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
590 { do_bad, SIGBUS, 0, "external abort on translation" },
591 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
592 { do_bad, SIGBUS, 0, "external abort on translation" },
593 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
594 { do_bad, SIGBUS, 0, "unknown 16" },
595 { do_bad, SIGBUS, 0, "unknown 17" },
596 { do_bad, SIGBUS, 0, "unknown 18" },
597 { do_bad, SIGBUS, 0, "unknown 19" },
598 { do_bad, SIGBUS, 0, "unknown 20" },
599 { do_bad, SIGBUS, 0, "unknown 21" },
600 { do_bad, SIGBUS, 0, "unknown 22" },
601 { do_bad, SIGBUS, 0, "unknown 23" },
602 { do_bad, SIGBUS, 0, "unknown 24" },
603 { do_bad, SIGBUS, 0, "unknown 25" },
604 { do_bad, SIGBUS, 0, "unknown 26" },
605 { do_bad, SIGBUS, 0, "unknown 27" },
606 { do_bad, SIGBUS, 0, "unknown 28" },
607 { do_bad, SIGBUS, 0, "unknown 29" },
608 { do_bad, SIGBUS, 0, "unknown 30" },
609 { do_bad, SIGBUS, 0, "unknown 31" },
610};
611
612void __init 557void __init
613hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), 558hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
614 int sig, int code, const char *name) 559 int sig, int code, const char *name)
@@ -641,6 +586,7 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
641 arm_notify_die("", regs, &info, ifsr, 0); 586 arm_notify_die("", regs, &info, ifsr, 0);
642} 587}
643 588
589#ifndef CONFIG_ARM_LPAE
644static int __init exceptions_init(void) 590static int __init exceptions_init(void)
645{ 591{
646 if (cpu_architecture() >= CPU_ARCH_ARMv6) { 592 if (cpu_architecture() >= CPU_ARCH_ARMv6) {
@@ -663,3 +609,4 @@ static int __init exceptions_init(void)
663} 609}
664 610
665arch_initcall(exceptions_init); 611arch_initcall(exceptions_init);
612#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-24 21:29:14 -0500