aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/kernel')
-rw-r--r--arch/powerpc/kernel/Makefile1
-rw-r--r--arch/powerpc/kernel/asm-offsets.c4
-rw-r--r--arch/powerpc/kernel/cpu_setup_power.S38
-rw-r--r--arch/powerpc/kernel/cputable.c16
-rw-r--r--arch/powerpc/kernel/dma-iommu.c4
-rw-r--r--arch/powerpc/kernel/eeh.c2
-rw-r--r--arch/powerpc/kernel/eeh_driver.c2
-rw-r--r--arch/powerpc/kernel/entry_64.S5
-rw-r--r--arch/powerpc/kernel/exceptions-64s.S242
-rw-r--r--arch/powerpc/kernel/idle_power7.S1
-rw-r--r--arch/powerpc/kernel/iommu.c136
-rw-r--r--arch/powerpc/kernel/irq.c12
-rw-r--r--arch/powerpc/kernel/mce.c345
-rw-r--r--arch/powerpc/kernel/mce_power.c284
-rw-r--r--arch/powerpc/kernel/misc_32.S4
-rw-r--r--arch/powerpc/kernel/misc_64.S6
-rw-r--r--arch/powerpc/kernel/paca.c32
-rw-r--r--arch/powerpc/kernel/setup_64.c16
-rw-r--r--arch/powerpc/kernel/smp.c9
-rw-r--r--arch/powerpc/kernel/sysfs.c72
-rw-r--r--arch/powerpc/kernel/time.c3
-rw-r--r--arch/powerpc/kernel/traps.c15
-rw-r--r--arch/powerpc/kernel/vio.c31
23 files changed, 1088 insertions, 192 deletions
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 445cb6e39d5b..904d713366ff 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -39,6 +39,7 @@ obj-$(CONFIG_PPC64) += setup_64.o sys_ppc32.o \
39obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o 39obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
40obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_ppc970.o cpu_setup_pa6t.o 40obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_ppc970.o cpu_setup_pa6t.o
41obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_power.o 41obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_power.o
42obj-$(CONFIG_PPC_BOOK3S_64) += mce.o mce_power.o
42obj64-$(CONFIG_RELOCATABLE) += reloc_64.o 43obj64-$(CONFIG_RELOCATABLE) += reloc_64.o
43obj-$(CONFIG_PPC_BOOK3E_64) += exceptions-64e.o idle_book3e.o 44obj-$(CONFIG_PPC_BOOK3E_64) += exceptions-64e.o idle_book3e.o
44obj-$(CONFIG_PPC_A2) += cpu_setup_a2.o 45obj-$(CONFIG_PPC_A2) += cpu_setup_a2.o
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 2ea5cc033ec8..41a283956a29 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -232,6 +232,10 @@ int main(void)
232 DEFINE(PACA_DTL_RIDX, offsetof(struct paca_struct, dtl_ridx)); 232 DEFINE(PACA_DTL_RIDX, offsetof(struct paca_struct, dtl_ridx));
233#endif /* CONFIG_PPC_STD_MMU_64 */ 233#endif /* CONFIG_PPC_STD_MMU_64 */
234 DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp)); 234 DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp));
235#ifdef CONFIG_PPC_BOOK3S_64
236 DEFINE(PACAMCEMERGSP, offsetof(struct paca_struct, mc_emergency_sp));
237 DEFINE(PACA_IN_MCE, offsetof(struct paca_struct, in_mce));
238#endif
235 DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id)); 239 DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id));
236 DEFINE(PACAKEXECSTATE, offsetof(struct paca_struct, kexec_state)); 240 DEFINE(PACAKEXECSTATE, offsetof(struct paca_struct, kexec_state));
237 DEFINE(PACA_STARTTIME, offsetof(struct paca_struct, starttime)); 241 DEFINE(PACA_STARTTIME, offsetof(struct paca_struct, starttime));
diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S
index 18b5b9cf8e37..37d1bb002aa9 100644
--- a/arch/powerpc/kernel/cpu_setup_power.S
+++ b/arch/powerpc/kernel/cpu_setup_power.S
@@ -29,7 +29,7 @@ _GLOBAL(__setup_cpu_power7)
29 mtspr SPRN_LPID,r0 29 mtspr SPRN_LPID,r0
30 mfspr r3,SPRN_LPCR 30 mfspr r3,SPRN_LPCR
31 bl __init_LPCR 31 bl __init_LPCR
32 bl __init_TLB 32 bl __init_tlb_power7
33 mtlr r11 33 mtlr r11
34 blr 34 blr
35 35
@@ -42,7 +42,7 @@ _GLOBAL(__restore_cpu_power7)
42 mtspr SPRN_LPID,r0 42 mtspr SPRN_LPID,r0
43 mfspr r3,SPRN_LPCR 43 mfspr r3,SPRN_LPCR
44 bl __init_LPCR 44 bl __init_LPCR
45 bl __init_TLB 45 bl __init_tlb_power7
46 mtlr r11 46 mtlr r11
47 blr 47 blr
48 48
@@ -59,7 +59,7 @@ _GLOBAL(__setup_cpu_power8)
59 oris r3, r3, LPCR_AIL_3@h 59 oris r3, r3, LPCR_AIL_3@h
60 bl __init_LPCR 60 bl __init_LPCR
61 bl __init_HFSCR 61 bl __init_HFSCR
62 bl __init_TLB 62 bl __init_tlb_power8
63 bl __init_PMU_HV 63 bl __init_PMU_HV
64 mtlr r11 64 mtlr r11
65 blr 65 blr
@@ -78,7 +78,7 @@ _GLOBAL(__restore_cpu_power8)
78 oris r3, r3, LPCR_AIL_3@h 78 oris r3, r3, LPCR_AIL_3@h
79 bl __init_LPCR 79 bl __init_LPCR
80 bl __init_HFSCR 80 bl __init_HFSCR
81 bl __init_TLB 81 bl __init_tlb_power8
82 bl __init_PMU_HV 82 bl __init_PMU_HV
83 mtlr r11 83 mtlr r11
84 blr 84 blr
@@ -134,15 +134,31 @@ __init_HFSCR:
134 mtspr SPRN_HFSCR,r3 134 mtspr SPRN_HFSCR,r3
135 blr 135 blr
136 136
137__init_TLB: 137/*
138 /* 138 * Clear the TLB using the specified IS form of tlbiel instruction
139 * Clear the TLB using the "IS 3" form of tlbiel instruction 139 * (invalidate by congruence class). P7 has 128 CCs., P8 has 512.
140 * (invalidate by congruence class). P7 has 128 CCs, P8 has 512 140 *
141 * so we just always do 512 141 * r3 = IS field
142 */ 142 */
143__init_tlb_power7:
144 li r3,0xc00 /* IS field = 0b11 */
145_GLOBAL(__flush_tlb_power7)
146 li r6,128
147 mtctr r6
148 mr r7,r3 /* IS field */
149 ptesync
1502: tlbiel r7
151 addi r7,r7,0x1000
152 bdnz 2b
153 ptesync
1541: blr
155
156__init_tlb_power8:
157 li r3,0xc00 /* IS field = 0b11 */
158_GLOBAL(__flush_tlb_power8)
143 li r6,512 159 li r6,512
144 mtctr r6 160 mtctr r6
145 li r7,0xc00 /* IS field = 0b11 */ 161 mr r7,r3 /* IS field */
146 ptesync 162 ptesync
1472: tlbiel r7 1632: tlbiel r7
148 addi r7,r7,0x1000 164 addi r7,r7,0x1000
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index 597d954e5860..6c8dd5da4de5 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -71,6 +71,10 @@ extern void __restore_cpu_power7(void);
71extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec); 71extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec);
72extern void __restore_cpu_power8(void); 72extern void __restore_cpu_power8(void);
73extern void __restore_cpu_a2(void); 73extern void __restore_cpu_a2(void);
74extern void __flush_tlb_power7(unsigned long inval_selector);
75extern void __flush_tlb_power8(unsigned long inval_selector);
76extern long __machine_check_early_realmode_p7(struct pt_regs *regs);
77extern long __machine_check_early_realmode_p8(struct pt_regs *regs);
74#endif /* CONFIG_PPC64 */ 78#endif /* CONFIG_PPC64 */
75#if defined(CONFIG_E500) 79#if defined(CONFIG_E500)
76extern void __setup_cpu_e5500(unsigned long offset, struct cpu_spec* spec); 80extern void __setup_cpu_e5500(unsigned long offset, struct cpu_spec* spec);
@@ -440,6 +444,8 @@ static struct cpu_spec __initdata cpu_specs[] = {
440 .oprofile_cpu_type = "ppc64/ibm-compat-v1", 444 .oprofile_cpu_type = "ppc64/ibm-compat-v1",
441 .cpu_setup = __setup_cpu_power7, 445 .cpu_setup = __setup_cpu_power7,
442 .cpu_restore = __restore_cpu_power7, 446 .cpu_restore = __restore_cpu_power7,
447 .flush_tlb = __flush_tlb_power7,
448 .machine_check_early = __machine_check_early_realmode_p7,
443 .platform = "power7", 449 .platform = "power7",
444 }, 450 },
445 { /* 2.07-compliant processor, i.e. Power8 "architected" mode */ 451 { /* 2.07-compliant processor, i.e. Power8 "architected" mode */
@@ -456,6 +462,8 @@ static struct cpu_spec __initdata cpu_specs[] = {
456 .oprofile_cpu_type = "ppc64/ibm-compat-v1", 462 .oprofile_cpu_type = "ppc64/ibm-compat-v1",
457 .cpu_setup = __setup_cpu_power8, 463 .cpu_setup = __setup_cpu_power8,
458 .cpu_restore = __restore_cpu_power8, 464 .cpu_restore = __restore_cpu_power8,
465 .flush_tlb = __flush_tlb_power8,
466 .machine_check_early = __machine_check_early_realmode_p8,
459 .platform = "power8", 467 .platform = "power8",
460 }, 468 },
461 { /* Power7 */ 469 { /* Power7 */
@@ -474,6 +482,8 @@ static struct cpu_spec __initdata cpu_specs[] = {
474 .oprofile_type = PPC_OPROFILE_POWER4, 482 .oprofile_type = PPC_OPROFILE_POWER4,
475 .cpu_setup = __setup_cpu_power7, 483 .cpu_setup = __setup_cpu_power7,
476 .cpu_restore = __restore_cpu_power7, 484 .cpu_restore = __restore_cpu_power7,
485 .flush_tlb = __flush_tlb_power7,
486 .machine_check_early = __machine_check_early_realmode_p7,
477 .platform = "power7", 487 .platform = "power7",
478 }, 488 },
479 { /* Power7+ */ 489 { /* Power7+ */
@@ -492,6 +502,8 @@ static struct cpu_spec __initdata cpu_specs[] = {
492 .oprofile_type = PPC_OPROFILE_POWER4, 502 .oprofile_type = PPC_OPROFILE_POWER4,
493 .cpu_setup = __setup_cpu_power7, 503 .cpu_setup = __setup_cpu_power7,
494 .cpu_restore = __restore_cpu_power7, 504 .cpu_restore = __restore_cpu_power7,
505 .flush_tlb = __flush_tlb_power7,
506 .machine_check_early = __machine_check_early_realmode_p7,
495 .platform = "power7+", 507 .platform = "power7+",
496 }, 508 },
497 { /* Power8E */ 509 { /* Power8E */
@@ -510,6 +522,8 @@ static struct cpu_spec __initdata cpu_specs[] = {
510 .oprofile_type = PPC_OPROFILE_INVALID, 522 .oprofile_type = PPC_OPROFILE_INVALID,
511 .cpu_setup = __setup_cpu_power8, 523 .cpu_setup = __setup_cpu_power8,
512 .cpu_restore = __restore_cpu_power8, 524 .cpu_restore = __restore_cpu_power8,
525 .flush_tlb = __flush_tlb_power8,
526 .machine_check_early = __machine_check_early_realmode_p8,
513 .platform = "power8", 527 .platform = "power8",
514 }, 528 },
515 { /* Power8 */ 529 { /* Power8 */
@@ -528,6 +542,8 @@ static struct cpu_spec __initdata cpu_specs[] = {
528 .oprofile_type = PPC_OPROFILE_INVALID, 542 .oprofile_type = PPC_OPROFILE_INVALID,
529 .cpu_setup = __setup_cpu_power8, 543 .cpu_setup = __setup_cpu_power8,
530 .cpu_restore = __restore_cpu_power8, 544 .cpu_restore = __restore_cpu_power8,
545 .flush_tlb = __flush_tlb_power8,
546 .machine_check_early = __machine_check_early_realmode_p8,
531 .platform = "power8", 547 .platform = "power8",
532 }, 548 },
533 { /* Cell Broadband Engine */ 549 { /* Cell Broadband Engine */
diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c
index e4897523de41..54d0116256f7 100644
--- a/arch/powerpc/kernel/dma-iommu.c
+++ b/arch/powerpc/kernel/dma-iommu.c
@@ -83,10 +83,10 @@ static int dma_iommu_dma_supported(struct device *dev, u64 mask)
83 return 0; 83 return 0;
84 } 84 }
85 85
86 if (tbl->it_offset > (mask >> IOMMU_PAGE_SHIFT)) { 86 if (tbl->it_offset > (mask >> tbl->it_page_shift)) {
87 dev_info(dev, "Warning: IOMMU offset too big for device mask\n"); 87 dev_info(dev, "Warning: IOMMU offset too big for device mask\n");
88 dev_info(dev, "mask: 0x%08llx, table offset: 0x%08lx\n", 88 dev_info(dev, "mask: 0x%08llx, table offset: 0x%08lx\n",
89 mask, tbl->it_offset << IOMMU_PAGE_SHIFT); 89 mask, tbl->it_offset << tbl->it_page_shift);
90 return 0; 90 return 0;
91 } else 91 } else
92 return 1; 92 return 1;
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index 4bd687d5e7aa..f4b7a227f183 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -84,7 +84,7 @@
84#define EEH_MAX_FAILS 2100000 84#define EEH_MAX_FAILS 2100000
85 85
86/* Time to wait for a PCI slot to report status, in milliseconds */ 86/* Time to wait for a PCI slot to report status, in milliseconds */
87#define PCI_BUS_RESET_WAIT_MSEC (60*1000) 87#define PCI_BUS_RESET_WAIT_MSEC (5*60*1000)
88 88
89/* Platform dependent EEH operations */ 89/* Platform dependent EEH operations */
90struct eeh_ops *eeh_ops = NULL; 90struct eeh_ops *eeh_ops = NULL;
diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c
index 36bed5a12750..4ef59c33777f 100644
--- a/arch/powerpc/kernel/eeh_driver.c
+++ b/arch/powerpc/kernel/eeh_driver.c
@@ -468,7 +468,7 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)
468/* The longest amount of time to wait for a pci device 468/* The longest amount of time to wait for a pci device
469 * to come back on line, in seconds. 469 * to come back on line, in seconds.
470 */ 470 */
471#define MAX_WAIT_FOR_RECOVERY 150 471#define MAX_WAIT_FOR_RECOVERY 300
472 472
473static void eeh_handle_normal_event(struct eeh_pe *pe) 473static void eeh_handle_normal_event(struct eeh_pe *pe)
474{ 474{
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index bbfb0294b354..770d6d65c47b 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -184,6 +184,11 @@ syscall_exit:
184 bl .do_show_syscall_exit 184 bl .do_show_syscall_exit
185 ld r3,RESULT(r1) 185 ld r3,RESULT(r1)
186#endif 186#endif
187#ifdef CONFIG_PPC_BOOK3S_64
188BEGIN_FTR_SECTION
189 bl .machine_check_process_queued_event
190END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
191#endif
187 CURRENT_THREAD_INFO(r12, r1) 192 CURRENT_THREAD_INFO(r12, r1)
188 193
189 ld r8,_MSR(r1) 194 ld r8,_MSR(r1)
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 9f905e40922e..38d507306a11 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -155,8 +155,30 @@ machine_check_pSeries_1:
155 */ 155 */
156 HMT_MEDIUM_PPR_DISCARD 156 HMT_MEDIUM_PPR_DISCARD
157 SET_SCRATCH0(r13) /* save r13 */ 157 SET_SCRATCH0(r13) /* save r13 */
158#ifdef CONFIG_PPC_P7_NAP
159BEGIN_FTR_SECTION
160 /* Running native on arch 2.06 or later, check if we are
161 * waking up from nap. We only handle no state loss and
162 * supervisor state loss. We do -not- handle hypervisor
163 * state loss at this time.
164 */
165 mfspr r13,SPRN_SRR1
166 rlwinm. r13,r13,47-31,30,31
167 beq 9f
168
169 /* waking up from powersave (nap) state */
170 cmpwi cr1,r13,2
171 /* Total loss of HV state is fatal. let's just stay stuck here */
172 bgt cr1,.
1739:
174END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
175#endif /* CONFIG_PPC_P7_NAP */
158 EXCEPTION_PROLOG_0(PACA_EXMC) 176 EXCEPTION_PROLOG_0(PACA_EXMC)
177BEGIN_FTR_SECTION
178 b machine_check_pSeries_early
179FTR_SECTION_ELSE
159 b machine_check_pSeries_0 180 b machine_check_pSeries_0
181ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
160 182
161 . = 0x300 183 . = 0x300
162 .globl data_access_pSeries 184 .globl data_access_pSeries
@@ -405,6 +427,64 @@ denorm_exception_hv:
405 427
406 .align 7 428 .align 7
407 /* moved from 0x200 */ 429 /* moved from 0x200 */
430machine_check_pSeries_early:
431BEGIN_FTR_SECTION
432 EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200)
433 /*
434 * Register contents:
435 * R13 = PACA
436 * R9 = CR
437 * Original R9 to R13 is saved on PACA_EXMC
438 *
439 * Switch to mc_emergency stack and handle re-entrancy (though we
440 * currently don't test for overflow). Save MCE registers srr1,
441 * srr0, dar and dsisr and then set ME=1
442 *
443 * We use paca->in_mce to check whether this is the first entry or
444 * nested machine check. We increment paca->in_mce to track nested
445 * machine checks.
446 *
447 * If this is the first entry then set stack pointer to
448 * paca->mc_emergency_sp, otherwise r1 is already pointing to
449 * stack frame on mc_emergency stack.
450 *
451 * NOTE: We are here with MSR_ME=0 (off), which means we risk a
452 * checkstop if we get another machine check exception before we do
453 * rfid with MSR_ME=1.
454 */
455 mr r11,r1 /* Save r1 */
456 lhz r10,PACA_IN_MCE(r13)
457 cmpwi r10,0 /* Are we in nested machine check */
458 bne 0f /* Yes, we are. */
459 /* First machine check entry */
460 ld r1,PACAMCEMERGSP(r13) /* Use MC emergency stack */
4610: subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
462 addi r10,r10,1 /* increment paca->in_mce */
463 sth r10,PACA_IN_MCE(r13)
464 std r11,GPR1(r1) /* Save r1 on the stack. */
465 std r11,0(r1) /* make stack chain pointer */
466 mfspr r11,SPRN_SRR0 /* Save SRR0 */
467 std r11,_NIP(r1)
468 mfspr r11,SPRN_SRR1 /* Save SRR1 */
469 std r11,_MSR(r1)
470 mfspr r11,SPRN_DAR /* Save DAR */
471 std r11,_DAR(r1)
472 mfspr r11,SPRN_DSISR /* Save DSISR */
473 std r11,_DSISR(r1)
474 std r9,_CCR(r1) /* Save CR in stackframe */
475 /* Save r9 through r13 from EXMC save area to stack frame. */
476 EXCEPTION_PROLOG_COMMON_2(PACA_EXMC)
477 mfmsr r11 /* get MSR value */
478 ori r11,r11,MSR_ME /* turn on ME bit */
479 ori r11,r11,MSR_RI /* turn on RI bit */
480 ld r12,PACAKBASE(r13) /* get high part of &label */
481 LOAD_HANDLER(r12, machine_check_handle_early)
482 mtspr SPRN_SRR0,r12
483 mtspr SPRN_SRR1,r11
484 rfid
485 b . /* prevent speculative execution */
486END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
487
408machine_check_pSeries: 488machine_check_pSeries:
409 .globl machine_check_fwnmi 489 .globl machine_check_fwnmi
410machine_check_fwnmi: 490machine_check_fwnmi:
@@ -688,30 +768,6 @@ kvmppc_skip_Hinterrupt:
688 768
689 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception) 769 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
690 770
691 /*
692 * Machine check is different because we use a different
693 * save area: PACA_EXMC instead of PACA_EXGEN.
694 */
695 .align 7
696 .globl machine_check_common
697machine_check_common:
698
699 mfspr r10,SPRN_DAR
700 std r10,PACA_EXGEN+EX_DAR(r13)
701 mfspr r10,SPRN_DSISR
702 stw r10,PACA_EXGEN+EX_DSISR(r13)
703 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
704 FINISH_NAP
705 DISABLE_INTS
706 ld r3,PACA_EXGEN+EX_DAR(r13)
707 lwz r4,PACA_EXGEN+EX_DSISR(r13)
708 std r3,_DAR(r1)
709 std r4,_DSISR(r1)
710 bl .save_nvgprs
711 addi r3,r1,STACK_FRAME_OVERHEAD
712 bl .machine_check_exception
713 b .ret_from_except
714
715 STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ) 771 STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
716 STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, .timer_interrupt) 772 STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, .timer_interrupt)
717 STD_EXCEPTION_COMMON(0x980, hdecrementer, .hdec_interrupt) 773 STD_EXCEPTION_COMMON(0x980, hdecrementer, .hdec_interrupt)
@@ -1080,6 +1136,30 @@ unrecov_user_slb:
1080#endif /* __DISABLED__ */ 1136#endif /* __DISABLED__ */
1081 1137
1082 1138
1139 /*
1140 * Machine check is different because we use a different
1141 * save area: PACA_EXMC instead of PACA_EXGEN.
1142 */
1143 .align 7
1144 .globl machine_check_common
1145machine_check_common:
1146
1147 mfspr r10,SPRN_DAR
1148 std r10,PACA_EXGEN+EX_DAR(r13)
1149 mfspr r10,SPRN_DSISR
1150 stw r10,PACA_EXGEN+EX_DSISR(r13)
1151 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
1152 FINISH_NAP
1153 DISABLE_INTS
1154 ld r3,PACA_EXGEN+EX_DAR(r13)
1155 lwz r4,PACA_EXGEN+EX_DSISR(r13)
1156 std r3,_DAR(r1)
1157 std r4,_DSISR(r1)
1158 bl .save_nvgprs
1159 addi r3,r1,STACK_FRAME_OVERHEAD
1160 bl .machine_check_exception
1161 b .ret_from_except
1162
1083 .align 7 1163 .align 7
1084 .globl alignment_common 1164 .globl alignment_common
1085alignment_common: 1165alignment_common:
@@ -1263,6 +1343,120 @@ _GLOBAL(opal_mc_secondary_handler)
1263#endif /* CONFIG_PPC_POWERNV */ 1343#endif /* CONFIG_PPC_POWERNV */
1264 1344
1265 1345
1346#define MACHINE_CHECK_HANDLER_WINDUP \
1347 /* Clear MSR_RI before setting SRR0 and SRR1. */\
1348 li r0,MSR_RI; \
1349 mfmsr r9; /* get MSR value */ \
1350 andc r9,r9,r0; \
1351 mtmsrd r9,1; /* Clear MSR_RI */ \
1352 /* Move original SRR0 and SRR1 into the respective regs */ \
1353 ld r9,_MSR(r1); \
1354 mtspr SPRN_SRR1,r9; \
1355 ld r3,_NIP(r1); \
1356 mtspr SPRN_SRR0,r3; \
1357 ld r9,_CTR(r1); \
1358 mtctr r9; \
1359 ld r9,_XER(r1); \
1360 mtxer r9; \
1361 ld r9,_LINK(r1); \
1362 mtlr r9; \
1363 REST_GPR(0, r1); \
1364 REST_8GPRS(2, r1); \
1365 REST_GPR(10, r1); \
1366 ld r11,_CCR(r1); \
1367 mtcr r11; \
1368 /* Decrement paca->in_mce. */ \
1369 lhz r12,PACA_IN_MCE(r13); \
1370 subi r12,r12,1; \
1371 sth r12,PACA_IN_MCE(r13); \
1372 REST_GPR(11, r1); \
1373 REST_2GPRS(12, r1); \
1374 /* restore original r1. */ \
1375 ld r1,GPR1(r1)
1376
1377 /*
1378 * Handle machine check early in real mode. We come here with
1379 * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
1380 */
1381 .align 7
1382 .globl machine_check_handle_early
1383machine_check_handle_early:
1384 std r0,GPR0(r1) /* Save r0 */
1385 EXCEPTION_PROLOG_COMMON_3(0x200)
1386 bl .save_nvgprs
1387 addi r3,r1,STACK_FRAME_OVERHEAD
1388 bl .machine_check_early
1389 ld r12,_MSR(r1)
1390#ifdef CONFIG_PPC_P7_NAP
1391 /*
1392 * Check if thread was in power saving mode. We come here when any
1393 * of the following is true:
1394 * a. thread wasn't in power saving mode
1395 * b. thread was in power saving mode with no state loss or
1396 * supervisor state loss
1397 *
1398 * Go back to nap again if (b) is true.
1399 */
1400 rlwinm. r11,r12,47-31,30,31 /* Was it in power saving mode? */
1401 beq 4f /* No, it wasn;t */
1402 /* Thread was in power saving mode. Go back to nap again. */
1403 cmpwi r11,2
1404 bne 3f
1405 /* Supervisor state loss */
1406 li r0,1
1407 stb r0,PACA_NAPSTATELOST(r13)
14083: bl .machine_check_queue_event
1409 MACHINE_CHECK_HANDLER_WINDUP
1410 GET_PACA(r13)
1411 ld r1,PACAR1(r13)
1412 b .power7_enter_nap_mode
14134:
1414#endif
1415 /*
1416 * Check if we are coming from hypervisor userspace. If yes then we
1417 * continue in host kernel in V mode to deliver the MC event.
1418 */
1419 rldicl. r11,r12,4,63 /* See if MC hit while in HV mode. */
1420 beq 5f
1421 andi. r11,r12,MSR_PR /* See if coming from user. */
1422 bne 9f /* continue in V mode if we are. */
1423
14245:
1425#ifdef CONFIG_KVM_BOOK3S_64_HV
1426 /*
1427 * We are coming from kernel context. Check if we are coming from
1428 * guest. if yes, then we can continue. We will fall through
1429 * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
1430 */
1431 lbz r11,HSTATE_IN_GUEST(r13)
1432 cmpwi r11,0 /* Check if coming from guest */
1433 bne 9f /* continue if we are. */
1434#endif
1435 /*
1436 * At this point we are not sure about what context we come from.
1437 * Queue up the MCE event and return from the interrupt.
1438 * But before that, check if this is an un-recoverable exception.
1439 * If yes, then stay on emergency stack and panic.
1440 */
1441 andi. r11,r12,MSR_RI
1442 bne 2f
14431: addi r3,r1,STACK_FRAME_OVERHEAD
1444 bl .unrecoverable_exception
1445 b 1b
14462:
1447 /*
1448 * Return from MC interrupt.
1449 * Queue up the MCE event so that we can log it later, while
1450 * returning from kernel or opal call.
1451 */
1452 bl .machine_check_queue_event
1453 MACHINE_CHECK_HANDLER_WINDUP
1454 rfid
14559:
1456 /* Deliver the machine check to host kernel in V mode. */
1457 MACHINE_CHECK_HANDLER_WINDUP
1458 b machine_check_pSeries
1459
1266/* 1460/*
1267 * r13 points to the PACA, r9 contains the saved CR, 1461 * r13 points to the PACA, r9 contains the saved CR,
1268 * r12 contain the saved SRR1, SRR0 is still ready for return 1462 * r12 contain the saved SRR1, SRR0 is still ready for return
diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S
index 847e40e62fce..3fdef0f0c67f 100644
--- a/arch/powerpc/kernel/idle_power7.S
+++ b/arch/powerpc/kernel/idle_power7.S
@@ -84,6 +84,7 @@ _GLOBAL(power7_nap)
84 std r9,_MSR(r1) 84 std r9,_MSR(r1)
85 std r1,PACAR1(r13) 85 std r1,PACAR1(r13)
86 86
87_GLOBAL(power7_enter_nap_mode)
87#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE 88#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
88 /* Tell KVM we're napping */ 89 /* Tell KVM we're napping */
89 li r4,KVM_HWTHREAD_IN_NAP 90 li r4,KVM_HWTHREAD_IN_NAP
diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c
index 572bb5b95f35..f58d8135aab2 100644
--- a/arch/powerpc/kernel/iommu.c
+++ b/arch/powerpc/kernel/iommu.c
@@ -251,14 +251,13 @@ again:
251 251
252 if (dev) 252 if (dev)
253 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 253 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
254 1 << IOMMU_PAGE_SHIFT); 254 1 << tbl->it_page_shift);
255 else 255 else
256 boundary_size = ALIGN(1UL << 32, 1 << IOMMU_PAGE_SHIFT); 256 boundary_size = ALIGN(1UL << 32, 1 << tbl->it_page_shift);
257 /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */ 257 /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */
258 258
259 n = iommu_area_alloc(tbl->it_map, limit, start, npages, 259 n = iommu_area_alloc(tbl->it_map, limit, start, npages, tbl->it_offset,
260 tbl->it_offset, boundary_size >> IOMMU_PAGE_SHIFT, 260 boundary_size >> tbl->it_page_shift, align_mask);
261 align_mask);
262 if (n == -1) { 261 if (n == -1) {
263 if (likely(pass == 0)) { 262 if (likely(pass == 0)) {
264 /* First try the pool from the start */ 263 /* First try the pool from the start */
@@ -320,12 +319,12 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
320 return DMA_ERROR_CODE; 319 return DMA_ERROR_CODE;
321 320
322 entry += tbl->it_offset; /* Offset into real TCE table */ 321 entry += tbl->it_offset; /* Offset into real TCE table */
323 ret = entry << IOMMU_PAGE_SHIFT; /* Set the return dma address */ 322 ret = entry << tbl->it_page_shift; /* Set the return dma address */
324 323
325 /* Put the TCEs in the HW table */ 324 /* Put the TCEs in the HW table */
326 build_fail = ppc_md.tce_build(tbl, entry, npages, 325 build_fail = ppc_md.tce_build(tbl, entry, npages,
327 (unsigned long)page & IOMMU_PAGE_MASK, 326 (unsigned long)page &
328 direction, attrs); 327 IOMMU_PAGE_MASK(tbl), direction, attrs);
329 328
330 /* ppc_md.tce_build() only returns non-zero for transient errors. 329 /* ppc_md.tce_build() only returns non-zero for transient errors.
331 * Clean up the table bitmap in this case and return 330 * Clean up the table bitmap in this case and return
@@ -352,7 +351,7 @@ static bool iommu_free_check(struct iommu_table *tbl, dma_addr_t dma_addr,
352{ 351{
353 unsigned long entry, free_entry; 352 unsigned long entry, free_entry;
354 353
355 entry = dma_addr >> IOMMU_PAGE_SHIFT; 354 entry = dma_addr >> tbl->it_page_shift;
356 free_entry = entry - tbl->it_offset; 355 free_entry = entry - tbl->it_offset;
357 356
358 if (((free_entry + npages) > tbl->it_size) || 357 if (((free_entry + npages) > tbl->it_size) ||
@@ -401,7 +400,7 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
401 unsigned long flags; 400 unsigned long flags;
402 struct iommu_pool *pool; 401 struct iommu_pool *pool;
403 402
404 entry = dma_addr >> IOMMU_PAGE_SHIFT; 403 entry = dma_addr >> tbl->it_page_shift;
405 free_entry = entry - tbl->it_offset; 404 free_entry = entry - tbl->it_offset;
406 405
407 pool = get_pool(tbl, free_entry); 406 pool = get_pool(tbl, free_entry);
@@ -468,13 +467,13 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
468 } 467 }
469 /* Allocate iommu entries for that segment */ 468 /* Allocate iommu entries for that segment */
470 vaddr = (unsigned long) sg_virt(s); 469 vaddr = (unsigned long) sg_virt(s);
471 npages = iommu_num_pages(vaddr, slen, IOMMU_PAGE_SIZE); 470 npages = iommu_num_pages(vaddr, slen, IOMMU_PAGE_SIZE(tbl));
472 align = 0; 471 align = 0;
473 if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && slen >= PAGE_SIZE && 472 if (tbl->it_page_shift < PAGE_SHIFT && slen >= PAGE_SIZE &&
474 (vaddr & ~PAGE_MASK) == 0) 473 (vaddr & ~PAGE_MASK) == 0)
475 align = PAGE_SHIFT - IOMMU_PAGE_SHIFT; 474 align = PAGE_SHIFT - tbl->it_page_shift;
476 entry = iommu_range_alloc(dev, tbl, npages, &handle, 475 entry = iommu_range_alloc(dev, tbl, npages, &handle,
477 mask >> IOMMU_PAGE_SHIFT, align); 476 mask >> tbl->it_page_shift, align);
478 477
479 DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen); 478 DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen);
480 479
@@ -489,16 +488,16 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
489 488
490 /* Convert entry to a dma_addr_t */ 489 /* Convert entry to a dma_addr_t */
491 entry += tbl->it_offset; 490 entry += tbl->it_offset;
492 dma_addr = entry << IOMMU_PAGE_SHIFT; 491 dma_addr = entry << tbl->it_page_shift;
493 dma_addr |= (s->offset & ~IOMMU_PAGE_MASK); 492 dma_addr |= (s->offset & ~IOMMU_PAGE_MASK(tbl));
494 493
495 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n", 494 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n",
496 npages, entry, dma_addr); 495 npages, entry, dma_addr);
497 496
498 /* Insert into HW table */ 497 /* Insert into HW table */
499 build_fail = ppc_md.tce_build(tbl, entry, npages, 498 build_fail = ppc_md.tce_build(tbl, entry, npages,
500 vaddr & IOMMU_PAGE_MASK, 499 vaddr & IOMMU_PAGE_MASK(tbl),
501 direction, attrs); 500 direction, attrs);
502 if(unlikely(build_fail)) 501 if(unlikely(build_fail))
503 goto failure; 502 goto failure;
504 503
@@ -559,9 +558,9 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
559 if (s->dma_length != 0) { 558 if (s->dma_length != 0) {
560 unsigned long vaddr, npages; 559 unsigned long vaddr, npages;
561 560
562 vaddr = s->dma_address & IOMMU_PAGE_MASK; 561 vaddr = s->dma_address & IOMMU_PAGE_MASK(tbl);
563 npages = iommu_num_pages(s->dma_address, s->dma_length, 562 npages = iommu_num_pages(s->dma_address, s->dma_length,
564 IOMMU_PAGE_SIZE); 563 IOMMU_PAGE_SIZE(tbl));
565 __iommu_free(tbl, vaddr, npages); 564 __iommu_free(tbl, vaddr, npages);
566 s->dma_address = DMA_ERROR_CODE; 565 s->dma_address = DMA_ERROR_CODE;
567 s->dma_length = 0; 566 s->dma_length = 0;
@@ -592,7 +591,7 @@ void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist,
592 if (sg->dma_length == 0) 591 if (sg->dma_length == 0)
593 break; 592 break;
594 npages = iommu_num_pages(dma_handle, sg->dma_length, 593 npages = iommu_num_pages(dma_handle, sg->dma_length,
595 IOMMU_PAGE_SIZE); 594 IOMMU_PAGE_SIZE(tbl));
596 __iommu_free(tbl, dma_handle, npages); 595 __iommu_free(tbl, dma_handle, npages);
597 sg = sg_next(sg); 596 sg = sg_next(sg);
598 } 597 }
@@ -676,7 +675,7 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid)
676 set_bit(0, tbl->it_map); 675 set_bit(0, tbl->it_map);
677 676
678 /* We only split the IOMMU table if we have 1GB or more of space */ 677 /* We only split the IOMMU table if we have 1GB or more of space */
679 if ((tbl->it_size << IOMMU_PAGE_SHIFT) >= (1UL * 1024 * 1024 * 1024)) 678 if ((tbl->it_size << tbl->it_page_shift) >= (1UL * 1024 * 1024 * 1024))
680 tbl->nr_pools = IOMMU_NR_POOLS; 679 tbl->nr_pools = IOMMU_NR_POOLS;
681 else 680 else
682 tbl->nr_pools = 1; 681 tbl->nr_pools = 1;
@@ -768,16 +767,16 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl,
768 767
769 vaddr = page_address(page) + offset; 768 vaddr = page_address(page) + offset;
770 uaddr = (unsigned long)vaddr; 769 uaddr = (unsigned long)vaddr;
771 npages = iommu_num_pages(uaddr, size, IOMMU_PAGE_SIZE); 770 npages = iommu_num_pages(uaddr, size, IOMMU_PAGE_SIZE(tbl));
772 771
773 if (tbl) { 772 if (tbl) {
774 align = 0; 773 align = 0;
775 if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && size >= PAGE_SIZE && 774 if (tbl->it_page_shift < PAGE_SHIFT && size >= PAGE_SIZE &&
776 ((unsigned long)vaddr & ~PAGE_MASK) == 0) 775 ((unsigned long)vaddr & ~PAGE_MASK) == 0)
777 align = PAGE_SHIFT - IOMMU_PAGE_SHIFT; 776 align = PAGE_SHIFT - tbl->it_page_shift;
778 777
779 dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction, 778 dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction,
780 mask >> IOMMU_PAGE_SHIFT, align, 779 mask >> tbl->it_page_shift, align,
781 attrs); 780 attrs);
782 if (dma_handle == DMA_ERROR_CODE) { 781 if (dma_handle == DMA_ERROR_CODE) {
783 if (printk_ratelimit()) { 782 if (printk_ratelimit()) {
@@ -786,7 +785,7 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl,
786 npages); 785 npages);
787 } 786 }
788 } else 787 } else
789 dma_handle |= (uaddr & ~IOMMU_PAGE_MASK); 788 dma_handle |= (uaddr & ~IOMMU_PAGE_MASK(tbl));
790 } 789 }
791 790
792 return dma_handle; 791 return dma_handle;
@@ -801,7 +800,8 @@ void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle,
801 BUG_ON(direction == DMA_NONE); 800 BUG_ON(direction == DMA_NONE);
802 801
803 if (tbl) { 802 if (tbl) {
804 npages = iommu_num_pages(dma_handle, size, IOMMU_PAGE_SIZE); 803 npages = iommu_num_pages(dma_handle, size,
804 IOMMU_PAGE_SIZE(tbl));
805 iommu_free(tbl, dma_handle, npages); 805 iommu_free(tbl, dma_handle, npages);
806 } 806 }
807} 807}
@@ -845,10 +845,10 @@ void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl,
845 memset(ret, 0, size); 845 memset(ret, 0, size);
846 846
847 /* Set up tces to cover the allocated range */ 847 /* Set up tces to cover the allocated range */
848 nio_pages = size >> IOMMU_PAGE_SHIFT; 848 nio_pages = size >> tbl->it_page_shift;
849 io_order = get_iommu_order(size); 849 io_order = get_iommu_order(size, tbl);
850 mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL, 850 mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
851 mask >> IOMMU_PAGE_SHIFT, io_order, NULL); 851 mask >> tbl->it_page_shift, io_order, NULL);
852 if (mapping == DMA_ERROR_CODE) { 852 if (mapping == DMA_ERROR_CODE) {
853 free_pages((unsigned long)ret, order); 853 free_pages((unsigned long)ret, order);
854 return NULL; 854 return NULL;
@@ -864,7 +864,7 @@ void iommu_free_coherent(struct iommu_table *tbl, size_t size,
864 unsigned int nio_pages; 864 unsigned int nio_pages;
865 865
866 size = PAGE_ALIGN(size); 866 size = PAGE_ALIGN(size);
867 nio_pages = size >> IOMMU_PAGE_SHIFT; 867 nio_pages = size >> tbl->it_page_shift;
868 iommu_free(tbl, dma_handle, nio_pages); 868 iommu_free(tbl, dma_handle, nio_pages);
869 size = PAGE_ALIGN(size); 869 size = PAGE_ALIGN(size);
870 free_pages((unsigned long)vaddr, get_order(size)); 870 free_pages((unsigned long)vaddr, get_order(size));
@@ -935,10 +935,10 @@ int iommu_tce_clear_param_check(struct iommu_table *tbl,
935 if (tce_value) 935 if (tce_value)
936 return -EINVAL; 936 return -EINVAL;
937 937
938 if (ioba & ~IOMMU_PAGE_MASK) 938 if (ioba & ~IOMMU_PAGE_MASK(tbl))
939 return -EINVAL; 939 return -EINVAL;
940 940
941 ioba >>= IOMMU_PAGE_SHIFT; 941 ioba >>= tbl->it_page_shift;
942 if (ioba < tbl->it_offset) 942 if (ioba < tbl->it_offset)
943 return -EINVAL; 943 return -EINVAL;
944 944
@@ -955,13 +955,13 @@ int iommu_tce_put_param_check(struct iommu_table *tbl,
955 if (!(tce & (TCE_PCI_WRITE | TCE_PCI_READ))) 955 if (!(tce & (TCE_PCI_WRITE | TCE_PCI_READ)))
956 return -EINVAL; 956 return -EINVAL;
957 957
958 if (tce & ~(IOMMU_PAGE_MASK | TCE_PCI_WRITE | TCE_PCI_READ)) 958 if (tce & ~(IOMMU_PAGE_MASK(tbl) | TCE_PCI_WRITE | TCE_PCI_READ))
959 return -EINVAL; 959 return -EINVAL;
960 960
961 if (ioba & ~IOMMU_PAGE_MASK) 961 if (ioba & ~IOMMU_PAGE_MASK(tbl))
962 return -EINVAL; 962 return -EINVAL;
963 963
964 ioba >>= IOMMU_PAGE_SHIFT; 964 ioba >>= tbl->it_page_shift;
965 if (ioba < tbl->it_offset) 965 if (ioba < tbl->it_offset)
966 return -EINVAL; 966 return -EINVAL;
967 967
@@ -1037,7 +1037,7 @@ int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,
1037 1037
1038 /* if (unlikely(ret)) 1038 /* if (unlikely(ret))
1039 pr_err("iommu_tce: %s failed on hwaddr=%lx ioba=%lx kva=%lx ret=%d\n", 1039 pr_err("iommu_tce: %s failed on hwaddr=%lx ioba=%lx kva=%lx ret=%d\n",
1040 __func__, hwaddr, entry << IOMMU_PAGE_SHIFT, 1040 __func__, hwaddr, entry << IOMMU_PAGE_SHIFT(tbl),
1041 hwaddr, ret); */ 1041 hwaddr, ret); */
1042 1042
1043 return ret; 1043 return ret;
@@ -1049,14 +1049,14 @@ int iommu_put_tce_user_mode(struct iommu_table *tbl, unsigned long entry,
1049{ 1049{
1050 int ret; 1050 int ret;
1051 struct page *page = NULL; 1051 struct page *page = NULL;
1052 unsigned long hwaddr, offset = tce & IOMMU_PAGE_MASK & ~PAGE_MASK; 1052 unsigned long hwaddr, offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
1053 enum dma_data_direction direction = iommu_tce_direction(tce); 1053 enum dma_data_direction direction = iommu_tce_direction(tce);
1054 1054
1055 ret = get_user_pages_fast(tce & PAGE_MASK, 1, 1055 ret = get_user_pages_fast(tce & PAGE_MASK, 1,
1056 direction != DMA_TO_DEVICE, &page); 1056 direction != DMA_TO_DEVICE, &page);
1057 if (unlikely(ret != 1)) { 1057 if (unlikely(ret != 1)) {
1058 /* pr_err("iommu_tce: get_user_pages_fast failed tce=%lx ioba=%lx ret=%d\n", 1058 /* pr_err("iommu_tce: get_user_pages_fast failed tce=%lx ioba=%lx ret=%d\n",
1059 tce, entry << IOMMU_PAGE_SHIFT, ret); */ 1059 tce, entry << IOMMU_PAGE_SHIFT(tbl), ret); */
1060 return -EFAULT; 1060 return -EFAULT;
1061 } 1061 }
1062 hwaddr = (unsigned long) page_address(page) + offset; 1062 hwaddr = (unsigned long) page_address(page) + offset;
@@ -1067,7 +1067,7 @@ int iommu_put_tce_user_mode(struct iommu_table *tbl, unsigned long entry,
1067 1067
1068 if (ret < 0) 1068 if (ret < 0)
1069 pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%d\n", 1069 pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%d\n",
1070 __func__, entry << IOMMU_PAGE_SHIFT, tce, ret); 1070 __func__, entry << tbl->it_page_shift, tce, ret);
1071 1071
1072 return ret; 1072 return ret;
1073} 1073}
@@ -1105,7 +1105,7 @@ void iommu_release_ownership(struct iommu_table *tbl)
1105} 1105}
1106EXPORT_SYMBOL_GPL(iommu_release_ownership); 1106EXPORT_SYMBOL_GPL(iommu_release_ownership);
1107 1107
1108static int iommu_add_device(struct device *dev) 1108int iommu_add_device(struct device *dev)
1109{ 1109{
1110 struct iommu_table *tbl; 1110 struct iommu_table *tbl;
1111 int ret = 0; 1111 int ret = 0;
@@ -1127,6 +1127,12 @@ static int iommu_add_device(struct device *dev)
1127 pr_debug("iommu_tce: adding %s to iommu group %d\n", 1127 pr_debug("iommu_tce: adding %s to iommu group %d\n",
1128 dev_name(dev), iommu_group_id(tbl->it_group)); 1128 dev_name(dev), iommu_group_id(tbl->it_group));
1129 1129
1130 if (PAGE_SIZE < IOMMU_PAGE_SIZE(tbl)) {
1131 pr_err("iommu_tce: unsupported iommu page size.");
1132 pr_err("%s has not been added\n", dev_name(dev));
1133 return -EINVAL;
1134 }
1135
1130 ret = iommu_group_add_device(tbl->it_group, dev); 1136 ret = iommu_group_add_device(tbl->it_group, dev);
1131 if (ret < 0) 1137 if (ret < 0)
1132 pr_err("iommu_tce: %s has not been added, ret=%d\n", 1138 pr_err("iommu_tce: %s has not been added, ret=%d\n",
@@ -1134,52 +1140,12 @@ static int iommu_add_device(struct device *dev)
1134 1140
1135 return ret; 1141 return ret;
1136} 1142}
1143EXPORT_SYMBOL_GPL(iommu_add_device);
1137 1144
1138static void iommu_del_device(struct device *dev) 1145void iommu_del_device(struct device *dev)
1139{ 1146{
1140 iommu_group_remove_device(dev); 1147 iommu_group_remove_device(dev);
1141} 1148}
1142 1149EXPORT_SYMBOL_GPL(iommu_del_device);
1143static int iommu_bus_notifier(struct notifier_block *nb,
1144 unsigned long action, void *data)
1145{
1146 struct device *dev = data;
1147
1148 switch (action) {
1149 case BUS_NOTIFY_ADD_DEVICE:
1150 return iommu_add_device(dev);
1151 case BUS_NOTIFY_DEL_DEVICE:
1152 iommu_del_device(dev);
1153 return 0;
1154 default:
1155 return 0;
1156 }
1157}
1158
1159static struct notifier_block tce_iommu_bus_nb = {
1160 .notifier_call = iommu_bus_notifier,
1161};
1162
1163static int __init tce_iommu_init(void)
1164{
1165 struct pci_dev *pdev = NULL;
1166
1167 BUILD_BUG_ON(PAGE_SIZE < IOMMU_PAGE_SIZE);
1168
1169 for_each_pci_dev(pdev)
1170 iommu_add_device(&pdev->dev);
1171
1172 bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1173 return 0;
1174}
1175
1176subsys_initcall_sync(tce_iommu_init);
1177
1178#else
1179
1180void iommu_register_group(struct iommu_table *tbl,
1181 int pci_domain_number, unsigned long pe_num)
1182{
1183}
1184 1150
1185#endif /* CONFIG_IOMMU_API */ 1151#endif /* CONFIG_IOMMU_API */
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index ba0165615215..9729b23bfb0a 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -354,8 +354,13 @@ int arch_show_interrupts(struct seq_file *p, int prec)
354 354
355 seq_printf(p, "%*s: ", prec, "LOC"); 355 seq_printf(p, "%*s: ", prec, "LOC");
356 for_each_online_cpu(j) 356 for_each_online_cpu(j)
357 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs); 357 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_event);
358 seq_printf(p, " Local timer interrupts\n"); 358 seq_printf(p, " Local timer interrupts for timer event device\n");
359
360 seq_printf(p, "%*s: ", prec, "LOC");
361 for_each_online_cpu(j)
362 seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_others);
363 seq_printf(p, " Local timer interrupts for others\n");
359 364
360 seq_printf(p, "%*s: ", prec, "SPU"); 365 seq_printf(p, "%*s: ", prec, "SPU");
361 for_each_online_cpu(j) 366 for_each_online_cpu(j)
@@ -389,11 +394,12 @@ int arch_show_interrupts(struct seq_file *p, int prec)
389 */ 394 */
390u64 arch_irq_stat_cpu(unsigned int cpu) 395u64 arch_irq_stat_cpu(unsigned int cpu)
391{ 396{
392 u64 sum = per_cpu(irq_stat, cpu).timer_irqs; 397 u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event;
393 398
394 sum += per_cpu(irq_stat, cpu).pmu_irqs; 399 sum += per_cpu(irq_stat, cpu).pmu_irqs;
395 sum += per_cpu(irq_stat, cpu).mce_exceptions; 400 sum += per_cpu(irq_stat, cpu).mce_exceptions;
396 sum += per_cpu(irq_stat, cpu).spurious_irqs; 401 sum += per_cpu(irq_stat, cpu).spurious_irqs;
402 sum += per_cpu(irq_stat, cpu).timer_irqs_others;
397#ifdef CONFIG_PPC_DOORBELL 403#ifdef CONFIG_PPC_DOORBELL
398 sum += per_cpu(irq_stat, cpu).doorbell_irqs; 404 sum += per_cpu(irq_stat, cpu).doorbell_irqs;
399#endif 405#endif
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
new file mode 100644
index 000000000000..c0c52ec1fca7
--- /dev/null
+++ b/arch/powerpc/kernel/mce.c
@@ -0,0 +1,345 @@
1/*
2 * Machine check exception handling.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright 2013 IBM Corporation
19 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
20 */
21
22#undef DEBUG
23#define pr_fmt(fmt) "mce: " fmt
24
25#include <linux/types.h>
26#include <linux/ptrace.h>
27#include <linux/percpu.h>
28#include <linux/export.h>
29#include <asm/mce.h>
30
31static DEFINE_PER_CPU(int, mce_nest_count);
32static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event);
33
34/* Queue for delayed MCE events. */
35static DEFINE_PER_CPU(int, mce_queue_count);
36static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event_queue);
37
38static void mce_set_error_info(struct machine_check_event *mce,
39 struct mce_error_info *mce_err)
40{
41 mce->error_type = mce_err->error_type;
42 switch (mce_err->error_type) {
43 case MCE_ERROR_TYPE_UE:
44 mce->u.ue_error.ue_error_type = mce_err->u.ue_error_type;
45 break;
46 case MCE_ERROR_TYPE_SLB:
47 mce->u.slb_error.slb_error_type = mce_err->u.slb_error_type;
48 break;
49 case MCE_ERROR_TYPE_ERAT:
50 mce->u.erat_error.erat_error_type = mce_err->u.erat_error_type;
51 break;
52 case MCE_ERROR_TYPE_TLB:
53 mce->u.tlb_error.tlb_error_type = mce_err->u.tlb_error_type;
54 break;
55 case MCE_ERROR_TYPE_UNKNOWN:
56 default:
57 break;
58 }
59}
60
61/*
62 * Decode and save high level MCE information into per cpu buffer which
63 * is an array of machine_check_event structure.
64 */
65void save_mce_event(struct pt_regs *regs, long handled,
66 struct mce_error_info *mce_err,
67 uint64_t addr)
68{
69 uint64_t srr1;
70 int index = __get_cpu_var(mce_nest_count)++;
71 struct machine_check_event *mce = &__get_cpu_var(mce_event[index]);
72
73 /*
74 * Return if we don't have enough space to log mce event.
75 * mce_nest_count may go beyond MAX_MC_EVT but that's ok,
76 * the check below will stop buffer overrun.
77 */
78 if (index >= MAX_MC_EVT)
79 return;
80
81 /* Populate generic machine check info */
82 mce->version = MCE_V1;
83 mce->srr0 = regs->nip;
84 mce->srr1 = regs->msr;
85 mce->gpr3 = regs->gpr[3];
86 mce->in_use = 1;
87
88 mce->initiator = MCE_INITIATOR_CPU;
89 if (handled)
90 mce->disposition = MCE_DISPOSITION_RECOVERED;
91 else
92 mce->disposition = MCE_DISPOSITION_NOT_RECOVERED;
93 mce->severity = MCE_SEV_ERROR_SYNC;
94
95 srr1 = regs->msr;
96
97 /*
98 * Populate the mce error_type and type-specific error_type.
99 */
100 mce_set_error_info(mce, mce_err);
101
102 if (!addr)
103 return;
104
105 if (mce->error_type == MCE_ERROR_TYPE_TLB) {
106 mce->u.tlb_error.effective_address_provided = true;
107 mce->u.tlb_error.effective_address = addr;
108 } else if (mce->error_type == MCE_ERROR_TYPE_SLB) {
109 mce->u.slb_error.effective_address_provided = true;
110 mce->u.slb_error.effective_address = addr;
111 } else if (mce->error_type == MCE_ERROR_TYPE_ERAT) {
112 mce->u.erat_error.effective_address_provided = true;
113 mce->u.erat_error.effective_address = addr;
114 } else if (mce->error_type == MCE_ERROR_TYPE_UE) {
115 mce->u.ue_error.effective_address_provided = true;
116 mce->u.ue_error.effective_address = addr;
117 }
118 return;
119}
120
121/*
122 * get_mce_event:
123 * mce Pointer to machine_check_event structure to be filled.
124 * release Flag to indicate whether to free the event slot or not.
125 * 0 <= do not release the mce event. Caller will invoke
126 * release_mce_event() once event has been consumed.
127 * 1 <= release the slot.
128 *
129 * return 1 = success
130 * 0 = failure
131 *
132 * get_mce_event() will be called by platform specific machine check
133 * handle routine and in KVM.
134 * When we call get_mce_event(), we are still in interrupt context and
135 * preemption will not be scheduled until ret_from_expect() routine
136 * is called.
137 */
138int get_mce_event(struct machine_check_event *mce, bool release)
139{
140 int index = __get_cpu_var(mce_nest_count) - 1;
141 struct machine_check_event *mc_evt;
142 int ret = 0;
143
144 /* Sanity check */
145 if (index < 0)
146 return ret;
147
148 /* Check if we have MCE info to process. */
149 if (index < MAX_MC_EVT) {
150 mc_evt = &__get_cpu_var(mce_event[index]);
151 /* Copy the event structure and release the original */
152 if (mce)
153 *mce = *mc_evt;
154 if (release)
155 mc_evt->in_use = 0;
156 ret = 1;
157 }
158 /* Decrement the count to free the slot. */
159 if (release)
160 __get_cpu_var(mce_nest_count)--;
161
162 return ret;
163}
164
165void release_mce_event(void)
166{
167 get_mce_event(NULL, true);
168}
169
170/*
171 * Queue up the MCE event which then can be handled later.
172 */
173void machine_check_queue_event(void)
174{
175 int index;
176 struct machine_check_event evt;
177
178 if (!get_mce_event(&evt, MCE_EVENT_RELEASE))
179 return;
180
181 index = __get_cpu_var(mce_queue_count)++;
182 /* If queue is full, just return for now. */
183 if (index >= MAX_MC_EVT) {
184 __get_cpu_var(mce_queue_count)--;
185 return;
186 }
187 __get_cpu_var(mce_event_queue[index]) = evt;
188}
189
190/*
191 * process pending MCE event from the mce event queue. This function will be
192 * called during syscall exit.
193 */
194void machine_check_process_queued_event(void)
195{
196 int index;
197
198 preempt_disable();
199 /*
200 * For now just print it to console.
201 * TODO: log this error event to FSP or nvram.
202 */
203 while (__get_cpu_var(mce_queue_count) > 0) {
204 index = __get_cpu_var(mce_queue_count) - 1;
205 machine_check_print_event_info(
206 &__get_cpu_var(mce_event_queue[index]));
207 __get_cpu_var(mce_queue_count)--;
208 }
209 preempt_enable();
210}
211
212void machine_check_print_event_info(struct machine_check_event *evt)
213{
214 const char *level, *sevstr, *subtype;
215 static const char *mc_ue_types[] = {
216 "Indeterminate",
217 "Instruction fetch",
218 "Page table walk ifetch",
219 "Load/Store",
220 "Page table walk Load/Store",
221 };
222 static const char *mc_slb_types[] = {
223 "Indeterminate",
224 "Parity",
225 "Multihit",
226 };
227 static const char *mc_erat_types[] = {
228 "Indeterminate",
229 "Parity",
230 "Multihit",
231 };
232 static const char *mc_tlb_types[] = {
233 "Indeterminate",
234 "Parity",
235 "Multihit",
236 };
237
238 /* Print things out */
239 if (evt->version != MCE_V1) {
240 pr_err("Machine Check Exception, Unknown event version %d !\n",
241 evt->version);
242 return;
243 }
244 switch (evt->severity) {
245 case MCE_SEV_NO_ERROR:
246 level = KERN_INFO;
247 sevstr = "Harmless";
248 break;
249 case MCE_SEV_WARNING:
250 level = KERN_WARNING;
251 sevstr = "";
252 break;
253 case MCE_SEV_ERROR_SYNC:
254 level = KERN_ERR;
255 sevstr = "Severe";
256 break;
257 case MCE_SEV_FATAL:
258 default:
259 level = KERN_ERR;
260 sevstr = "Fatal";
261 break;
262 }
263
264 printk("%s%s Machine check interrupt [%s]\n", level, sevstr,
265 evt->disposition == MCE_DISPOSITION_RECOVERED ?
266 "Recovered" : "[Not recovered");
267 printk("%s Initiator: %s\n", level,
268 evt->initiator == MCE_INITIATOR_CPU ? "CPU" : "Unknown");
269 switch (evt->error_type) {
270 case MCE_ERROR_TYPE_UE:
271 subtype = evt->u.ue_error.ue_error_type <
272 ARRAY_SIZE(mc_ue_types) ?
273 mc_ue_types[evt->u.ue_error.ue_error_type]
274 : "Unknown";
275 printk("%s Error type: UE [%s]\n", level, subtype);
276 if (evt->u.ue_error.effective_address_provided)
277 printk("%s Effective address: %016llx\n",
278 level, evt->u.ue_error.effective_address);
279 if (evt->u.ue_error.physical_address_provided)
280 printk("%s Physial address: %016llx\n",
281 level, evt->u.ue_error.physical_address);
282 break;
283 case MCE_ERROR_TYPE_SLB:
284 subtype = evt->u.slb_error.slb_error_type <
285 ARRAY_SIZE(mc_slb_types) ?
286 mc_slb_types[evt->u.slb_error.slb_error_type]
287 : "Unknown";
288 printk("%s Error type: SLB [%s]\n", level, subtype);
289 if (evt->u.slb_error.effective_address_provided)
290 printk("%s Effective address: %016llx\n",
291 level, evt->u.slb_error.effective_address);
292 break;
293 case MCE_ERROR_TYPE_ERAT:
294 subtype = evt->u.erat_error.erat_error_type <
295 ARRAY_SIZE(mc_erat_types) ?
296 mc_erat_types[evt->u.erat_error.erat_error_type]
297 : "Unknown";
298 printk("%s Error type: ERAT [%s]\n", level, subtype);
299 if (evt->u.erat_error.effective_address_provided)
300 printk("%s Effective address: %016llx\n",
301 level, evt->u.erat_error.effective_address);
302 break;
303 case MCE_ERROR_TYPE_TLB:
304 subtype = evt->u.tlb_error.tlb_error_type <
305 ARRAY_SIZE(mc_tlb_types) ?
306 mc_tlb_types[evt->u.tlb_error.tlb_error_type]
307 : "Unknown";
308 printk("%s Error type: TLB [%s]\n", level, subtype);
309 if (evt->u.tlb_error.effective_address_provided)
310 printk("%s Effective address: %016llx\n",
311 level, evt->u.tlb_error.effective_address);
312 break;
313 default:
314 case MCE_ERROR_TYPE_UNKNOWN:
315 printk("%s Error type: Unknown\n", level);
316 break;
317 }
318}
319
320uint64_t get_mce_fault_addr(struct machine_check_event *evt)
321{
322 switch (evt->error_type) {
323 case MCE_ERROR_TYPE_UE:
324 if (evt->u.ue_error.effective_address_provided)
325 return evt->u.ue_error.effective_address;
326 break;
327 case MCE_ERROR_TYPE_SLB:
328 if (evt->u.slb_error.effective_address_provided)
329 return evt->u.slb_error.effective_address;
330 break;
331 case MCE_ERROR_TYPE_ERAT:
332 if (evt->u.erat_error.effective_address_provided)
333 return evt->u.erat_error.effective_address;
334 break;
335 case MCE_ERROR_TYPE_TLB:
336 if (evt->u.tlb_error.effective_address_provided)
337 return evt->u.tlb_error.effective_address;
338 break;
339 default:
340 case MCE_ERROR_TYPE_UNKNOWN:
341 break;
342 }
343 return 0;
344}
345EXPORT_SYMBOL(get_mce_fault_addr);
diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c
new file mode 100644
index 000000000000..27c93f41166f
--- /dev/null
+++ b/arch/powerpc/kernel/mce_power.c
@@ -0,0 +1,284 @@
1/*
2 * Machine check exception handling CPU-side for power7 and power8
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright 2013 IBM Corporation
19 * Author: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
20 */
21
22#undef DEBUG
23#define pr_fmt(fmt) "mce_power: " fmt
24
25#include <linux/types.h>
26#include <linux/ptrace.h>
27#include <asm/mmu.h>
28#include <asm/mce.h>
29
30/* flush SLBs and reload */
31static void flush_and_reload_slb(void)
32{
33 struct slb_shadow *slb;
34 unsigned long i, n;
35
36 /* Invalidate all SLBs */
37 asm volatile("slbmte %0,%0; slbia" : : "r" (0));
38
39#ifdef CONFIG_KVM_BOOK3S_HANDLER
40 /*
41 * If machine check is hit when in guest or in transition, we will
42 * only flush the SLBs and continue.
43 */
44 if (get_paca()->kvm_hstate.in_guest)
45 return;
46#endif
47
48 /* For host kernel, reload the SLBs from shadow SLB buffer. */
49 slb = get_slb_shadow();
50 if (!slb)
51 return;
52
53 n = min_t(u32, be32_to_cpu(slb->persistent), SLB_MIN_SIZE);
54
55 /* Load up the SLB entries from shadow SLB */
56 for (i = 0; i < n; i++) {
57 unsigned long rb = be64_to_cpu(slb->save_area[i].esid);
58 unsigned long rs = be64_to_cpu(slb->save_area[i].vsid);
59
60 rb = (rb & ~0xFFFul) | i;
61 asm volatile("slbmte %0,%1" : : "r" (rs), "r" (rb));
62 }
63}
64
65static long mce_handle_derror(uint64_t dsisr, uint64_t slb_error_bits)
66{
67 long handled = 1;
68
69 /*
70 * flush and reload SLBs for SLB errors and flush TLBs for TLB errors.
71 * reset the error bits whenever we handle them so that at the end
72 * we can check whether we handled all of them or not.
73 * */
74 if (dsisr & slb_error_bits) {
75 flush_and_reload_slb();
76 /* reset error bits */
77 dsisr &= ~(slb_error_bits);
78 }
79 if (dsisr & P7_DSISR_MC_TLB_MULTIHIT_MFTLB) {
80 if (cur_cpu_spec && cur_cpu_spec->flush_tlb)
81 cur_cpu_spec->flush_tlb(TLBIEL_INVAL_PAGE);
82 /* reset error bits */
83 dsisr &= ~P7_DSISR_MC_TLB_MULTIHIT_MFTLB;
84 }
85 /* Any other errors we don't understand? */
86 if (dsisr & 0xffffffffUL)
87 handled = 0;
88
89 return handled;
90}
91
92static long mce_handle_derror_p7(uint64_t dsisr)
93{
94 return mce_handle_derror(dsisr, P7_DSISR_MC_SLB_ERRORS);
95}
96
97static long mce_handle_common_ierror(uint64_t srr1)
98{
99 long handled = 0;
100
101 switch (P7_SRR1_MC_IFETCH(srr1)) {
102 case 0:
103 break;
104 case P7_SRR1_MC_IFETCH_SLB_PARITY:
105 case P7_SRR1_MC_IFETCH_SLB_MULTIHIT:
106 /* flush and reload SLBs for SLB errors. */
107 flush_and_reload_slb();
108 handled = 1;
109 break;
110 case P7_SRR1_MC_IFETCH_TLB_MULTIHIT:
111 if (cur_cpu_spec && cur_cpu_spec->flush_tlb) {
112 cur_cpu_spec->flush_tlb(TLBIEL_INVAL_PAGE);
113 handled = 1;
114 }
115 break;
116 default:
117 break;
118 }
119
120 return handled;
121}
122
123static long mce_handle_ierror_p7(uint64_t srr1)
124{
125 long handled = 0;
126
127 handled = mce_handle_common_ierror(srr1);
128
129 if (P7_SRR1_MC_IFETCH(srr1) == P7_SRR1_MC_IFETCH_SLB_BOTH) {
130 flush_and_reload_slb();
131 handled = 1;
132 }
133 return handled;
134}
135
136static void mce_get_common_ierror(struct mce_error_info *mce_err, uint64_t srr1)
137{
138 switch (P7_SRR1_MC_IFETCH(srr1)) {
139 case P7_SRR1_MC_IFETCH_SLB_PARITY:
140 mce_err->error_type = MCE_ERROR_TYPE_SLB;
141 mce_err->u.slb_error_type = MCE_SLB_ERROR_PARITY;
142 break;
143 case P7_SRR1_MC_IFETCH_SLB_MULTIHIT:
144 mce_err->error_type = MCE_ERROR_TYPE_SLB;
145 mce_err->u.slb_error_type = MCE_SLB_ERROR_MULTIHIT;
146 break;
147 case P7_SRR1_MC_IFETCH_TLB_MULTIHIT:
148 mce_err->error_type = MCE_ERROR_TYPE_TLB;
149 mce_err->u.tlb_error_type = MCE_TLB_ERROR_MULTIHIT;
150 break;
151 case P7_SRR1_MC_IFETCH_UE:
152 case P7_SRR1_MC_IFETCH_UE_IFU_INTERNAL:
153 mce_err->error_type = MCE_ERROR_TYPE_UE;
154 mce_err->u.ue_error_type = MCE_UE_ERROR_IFETCH;
155 break;
156 case P7_SRR1_MC_IFETCH_UE_TLB_RELOAD:
157 mce_err->error_type = MCE_ERROR_TYPE_UE;
158 mce_err->u.ue_error_type =
159 MCE_UE_ERROR_PAGE_TABLE_WALK_IFETCH;
160 break;
161 }
162}
163
164static void mce_get_ierror_p7(struct mce_error_info *mce_err, uint64_t srr1)
165{
166 mce_get_common_ierror(mce_err, srr1);
167 if (P7_SRR1_MC_IFETCH(srr1) == P7_SRR1_MC_IFETCH_SLB_BOTH) {
168 mce_err->error_type = MCE_ERROR_TYPE_SLB;
169 mce_err->u.slb_error_type = MCE_SLB_ERROR_INDETERMINATE;
170 }
171}
172
173static void mce_get_derror_p7(struct mce_error_info *mce_err, uint64_t dsisr)
174{
175 if (dsisr & P7_DSISR_MC_UE) {
176 mce_err->error_type = MCE_ERROR_TYPE_UE;
177 mce_err->u.ue_error_type = MCE_UE_ERROR_LOAD_STORE;
178 } else if (dsisr & P7_DSISR_MC_UE_TABLEWALK) {
179 mce_err->error_type = MCE_ERROR_TYPE_UE;
180 mce_err->u.ue_error_type =
181 MCE_UE_ERROR_PAGE_TABLE_WALK_LOAD_STORE;
182 } else if (dsisr & P7_DSISR_MC_ERAT_MULTIHIT) {
183 mce_err->error_type = MCE_ERROR_TYPE_ERAT;
184 mce_err->u.erat_error_type = MCE_ERAT_ERROR_MULTIHIT;
185 } else if (dsisr & P7_DSISR_MC_SLB_MULTIHIT) {
186 mce_err->error_type = MCE_ERROR_TYPE_SLB;
187 mce_err->u.slb_error_type = MCE_SLB_ERROR_MULTIHIT;
188 } else if (dsisr & P7_DSISR_MC_SLB_PARITY_MFSLB) {
189 mce_err->error_type = MCE_ERROR_TYPE_SLB;
190 mce_err->u.slb_error_type = MCE_SLB_ERROR_PARITY;
191 } else if (dsisr & P7_DSISR_MC_TLB_MULTIHIT_MFTLB) {
192 mce_err->error_type = MCE_ERROR_TYPE_TLB;
193 mce_err->u.tlb_error_type = MCE_TLB_ERROR_MULTIHIT;
194 } else if (dsisr & P7_DSISR_MC_SLB_MULTIHIT_PARITY) {
195 mce_err->error_type = MCE_ERROR_TYPE_SLB;
196 mce_err->u.slb_error_type = MCE_SLB_ERROR_INDETERMINATE;
197 }
198}
199
200long __machine_check_early_realmode_p7(struct pt_regs *regs)
201{
202 uint64_t srr1, addr;
203 long handled = 1;
204 struct mce_error_info mce_error_info = { 0 };
205
206 srr1 = regs->msr;
207
208 /*
209 * Handle memory errors depending whether this was a load/store or
210 * ifetch exception. Also, populate the mce error_type and
211 * type-specific error_type from either SRR1 or DSISR, depending
212 * whether this was a load/store or ifetch exception
213 */
214 if (P7_SRR1_MC_LOADSTORE(srr1)) {
215 handled = mce_handle_derror_p7(regs->dsisr);
216 mce_get_derror_p7(&mce_error_info, regs->dsisr);
217 addr = regs->dar;
218 } else {
219 handled = mce_handle_ierror_p7(srr1);
220 mce_get_ierror_p7(&mce_error_info, srr1);
221 addr = regs->nip;
222 }
223
224 save_mce_event(regs, handled, &mce_error_info, addr);
225 return handled;
226}
227
228static void mce_get_ierror_p8(struct mce_error_info *mce_err, uint64_t srr1)
229{
230 mce_get_common_ierror(mce_err, srr1);
231 if (P7_SRR1_MC_IFETCH(srr1) == P8_SRR1_MC_IFETCH_ERAT_MULTIHIT) {
232 mce_err->error_type = MCE_ERROR_TYPE_ERAT;
233 mce_err->u.erat_error_type = MCE_ERAT_ERROR_MULTIHIT;
234 }
235}
236
237static void mce_get_derror_p8(struct mce_error_info *mce_err, uint64_t dsisr)
238{
239 mce_get_derror_p7(mce_err, dsisr);
240 if (dsisr & P8_DSISR_MC_ERAT_MULTIHIT_SEC) {
241 mce_err->error_type = MCE_ERROR_TYPE_ERAT;
242 mce_err->u.erat_error_type = MCE_ERAT_ERROR_MULTIHIT;
243 }
244}
245
246static long mce_handle_ierror_p8(uint64_t srr1)
247{
248 long handled = 0;
249
250 handled = mce_handle_common_ierror(srr1);
251
252 if (P7_SRR1_MC_IFETCH(srr1) == P8_SRR1_MC_IFETCH_ERAT_MULTIHIT) {
253 flush_and_reload_slb();
254 handled = 1;
255 }
256 return handled;
257}
258
259static long mce_handle_derror_p8(uint64_t dsisr)
260{
261 return mce_handle_derror(dsisr, P8_DSISR_MC_SLB_ERRORS);
262}
263
264long __machine_check_early_realmode_p8(struct pt_regs *regs)
265{
266 uint64_t srr1, addr;
267 long handled = 1;
268 struct mce_error_info mce_error_info = { 0 };
269
270 srr1 = regs->msr;
271
272 if (P7_SRR1_MC_LOADSTORE(srr1)) {
273 handled = mce_handle_derror_p8(regs->dsisr);
274 mce_get_derror_p8(&mce_error_info, regs->dsisr);
275 addr = regs->dar;
276 } else {
277 handled = mce_handle_ierror_p8(srr1);
278 mce_get_ierror_p8(&mce_error_info, srr1);
279 addr = regs->nip;
280 }
281
282 save_mce_event(regs, handled, &mce_error_info, addr);
283 return handled;
284}
diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S
index e47d268727a4..879f09620f83 100644
--- a/arch/powerpc/kernel/misc_32.S
+++ b/arch/powerpc/kernel/misc_32.S
@@ -344,7 +344,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_UNIFIED_ID_CACHE)
344 */ 344 */
345_KPROBE(flush_icache_range) 345_KPROBE(flush_icache_range)
346BEGIN_FTR_SECTION 346BEGIN_FTR_SECTION
347 isync 347 PURGE_PREFETCHED_INS
348 blr /* for 601, do nothing */ 348 blr /* for 601, do nothing */
349END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 349END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
350 li r5,L1_CACHE_BYTES-1 350 li r5,L1_CACHE_BYTES-1
@@ -448,6 +448,7 @@ _GLOBAL(invalidate_dcache_range)
448 */ 448 */
449_GLOBAL(__flush_dcache_icache) 449_GLOBAL(__flush_dcache_icache)
450BEGIN_FTR_SECTION 450BEGIN_FTR_SECTION
451 PURGE_PREFETCHED_INS
451 blr 452 blr
452END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 453END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
453 rlwinm r3,r3,0,0,31-PAGE_SHIFT /* Get page base address */ 454 rlwinm r3,r3,0,0,31-PAGE_SHIFT /* Get page base address */
@@ -489,6 +490,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_44x)
489 */ 490 */
490_GLOBAL(__flush_dcache_icache_phys) 491_GLOBAL(__flush_dcache_icache_phys)
491BEGIN_FTR_SECTION 492BEGIN_FTR_SECTION
493 PURGE_PREFETCHED_INS
492 blr /* for 601, do nothing */ 494 blr /* for 601, do nothing */
493END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 495END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
494 mfmsr r10 496 mfmsr r10
diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S
index 64bf8db12b15..3d0249599d52 100644
--- a/arch/powerpc/kernel/misc_64.S
+++ b/arch/powerpc/kernel/misc_64.S
@@ -67,6 +67,7 @@ PPC64_CACHES:
67 67
68_KPROBE(flush_icache_range) 68_KPROBE(flush_icache_range)
69BEGIN_FTR_SECTION 69BEGIN_FTR_SECTION
70 PURGE_PREFETCHED_INS
70 blr 71 blr
71END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) 72END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
72/* 73/*
@@ -211,6 +212,11 @@ _GLOBAL(__flush_dcache_icache)
211 * Different systems have different cache line sizes 212 * Different systems have different cache line sizes
212 */ 213 */
213 214
215BEGIN_FTR_SECTION
216 PURGE_PREFETCHED_INS
217 blr
218END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
219
214/* Flush the dcache */ 220/* Flush the dcache */
215 ld r7,PPC64_CACHES@toc(r2) 221 ld r7,PPC64_CACHES@toc(r2)
216 clrrdi r3,r3,PAGE_SHIFT /* Page align */ 222 clrrdi r3,r3,PAGE_SHIFT /* Page align */
diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c
index 0620eaaaad45..623c356fe34f 100644
--- a/arch/powerpc/kernel/paca.c
+++ b/arch/powerpc/kernel/paca.c
@@ -99,12 +99,28 @@ static inline void free_lppacas(void) { }
99 * 3 persistent SLBs are registered here. The buffer will be zero 99 * 3 persistent SLBs are registered here. The buffer will be zero
100 * initially, hence will all be invaild until we actually write them. 100 * initially, hence will all be invaild until we actually write them.
101 */ 101 */
102struct slb_shadow slb_shadow[] __cacheline_aligned = { 102static struct slb_shadow *slb_shadow;
103 [0 ... (NR_CPUS-1)] = { 103
104 .persistent = cpu_to_be32(SLB_NUM_BOLTED), 104static void __init allocate_slb_shadows(int nr_cpus, int limit)
105 .buffer_length = cpu_to_be32(sizeof(struct slb_shadow)), 105{
106 }, 106 int size = PAGE_ALIGN(sizeof(struct slb_shadow) * nr_cpus);
107}; 107 slb_shadow = __va(memblock_alloc_base(size, PAGE_SIZE, limit));
108 memset(slb_shadow, 0, size);
109}
110
111static struct slb_shadow * __init init_slb_shadow(int cpu)
112{
113 struct slb_shadow *s = &slb_shadow[cpu];
114
115 s->persistent = cpu_to_be32(SLB_NUM_BOLTED);
116 s->buffer_length = cpu_to_be32(sizeof(*s));
117
118 return s;
119}
120
121#else /* CONFIG_PPC_STD_MMU_64 */
122
123static void __init allocate_slb_shadows(int nr_cpus, int limit) { }
108 124
109#endif /* CONFIG_PPC_STD_MMU_64 */ 125#endif /* CONFIG_PPC_STD_MMU_64 */
110 126
@@ -142,7 +158,7 @@ void __init initialise_paca(struct paca_struct *new_paca, int cpu)
142 new_paca->__current = &init_task; 158 new_paca->__current = &init_task;
143 new_paca->data_offset = 0xfeeeeeeeeeeeeeeeULL; 159 new_paca->data_offset = 0xfeeeeeeeeeeeeeeeULL;
144#ifdef CONFIG_PPC_STD_MMU_64 160#ifdef CONFIG_PPC_STD_MMU_64
145 new_paca->slb_shadow_ptr = &slb_shadow[cpu]; 161 new_paca->slb_shadow_ptr = init_slb_shadow(cpu);
146#endif /* CONFIG_PPC_STD_MMU_64 */ 162#endif /* CONFIG_PPC_STD_MMU_64 */
147} 163}
148 164
@@ -190,6 +206,8 @@ void __init allocate_pacas(void)
190 206
191 allocate_lppacas(nr_cpu_ids, limit); 207 allocate_lppacas(nr_cpu_ids, limit);
192 208
209 allocate_slb_shadows(nr_cpu_ids, limit);
210
193 /* Can't use for_each_*_cpu, as they aren't functional yet */ 211 /* Can't use for_each_*_cpu, as they aren't functional yet */
194 for (cpu = 0; cpu < nr_cpu_ids; cpu++) 212 for (cpu = 0; cpu < nr_cpu_ids; cpu++)
195 initialise_paca(&paca[cpu], cpu); 213 initialise_paca(&paca[cpu], cpu);
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 4085aaa9478f..2232aff66059 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -520,9 +520,6 @@ static void __init irqstack_early_init(void)
520#ifdef CONFIG_PPC_BOOK3E 520#ifdef CONFIG_PPC_BOOK3E
521static void __init exc_lvl_early_init(void) 521static void __init exc_lvl_early_init(void)
522{ 522{
523 extern unsigned int interrupt_base_book3e;
524 extern unsigned int exc_debug_debug_book3e;
525
526 unsigned int i; 523 unsigned int i;
527 524
528 for_each_possible_cpu(i) { 525 for_each_possible_cpu(i) {
@@ -535,8 +532,7 @@ static void __init exc_lvl_early_init(void)
535 } 532 }
536 533
537 if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) 534 if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
538 patch_branch(&interrupt_base_book3e + (0x040 / 4) + 1, 535 patch_exception(0x040, exc_debug_debug_book3e);
539 (unsigned long)&exc_debug_debug_book3e, 0);
540} 536}
541#else 537#else
542#define exc_lvl_early_init() 538#define exc_lvl_early_init()
@@ -544,7 +540,8 @@ static void __init exc_lvl_early_init(void)
544 540
545/* 541/*
546 * Stack space used when we detect a bad kernel stack pointer, and 542 * Stack space used when we detect a bad kernel stack pointer, and
547 * early in SMP boots before relocation is enabled. 543 * early in SMP boots before relocation is enabled. Exclusive emergency
544 * stack for machine checks.
548 */ 545 */
549static void __init emergency_stack_init(void) 546static void __init emergency_stack_init(void)
550{ 547{
@@ -567,6 +564,13 @@ static void __init emergency_stack_init(void)
567 sp = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit); 564 sp = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit);
568 sp += THREAD_SIZE; 565 sp += THREAD_SIZE;
569 paca[i].emergency_sp = __va(sp); 566 paca[i].emergency_sp = __va(sp);
567
568#ifdef CONFIG_PPC_BOOK3S_64
569 /* emergency stack for machine check exception handling. */
570 sp = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit);
571 sp += THREAD_SIZE;
572 paca[i].mc_emergency_sp = __va(sp);
573#endif
570 } 574 }
571} 575}
572 576
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index c1cf4a1522d9..ac2621af3154 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -369,13 +369,8 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
369 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid)); 369 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
370 cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid)); 370 cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
371 371
372 if (smp_ops) 372 if (smp_ops && smp_ops->probe)
373 if (smp_ops->probe) 373 smp_ops->probe();
374 max_cpus = smp_ops->probe();
375 else
376 max_cpus = NR_CPUS;
377 else
378 max_cpus = 1;
379} 374}
380 375
381void smp_prepare_boot_cpu(void) 376void smp_prepare_boot_cpu(void)
diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index b4e667663d9b..cad777eb613a 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -108,14 +108,14 @@ void ppc_enable_pmcs(void)
108} 108}
109EXPORT_SYMBOL(ppc_enable_pmcs); 109EXPORT_SYMBOL(ppc_enable_pmcs);
110 110
111#define SYSFS_PMCSETUP(NAME, ADDRESS) \ 111#define __SYSFS_SPRSETUP(NAME, ADDRESS, EXTRA) \
112static void read_##NAME(void *val) \ 112static void read_##NAME(void *val) \
113{ \ 113{ \
114 *(unsigned long *)val = mfspr(ADDRESS); \ 114 *(unsigned long *)val = mfspr(ADDRESS); \
115} \ 115} \
116static void write_##NAME(void *val) \ 116static void write_##NAME(void *val) \
117{ \ 117{ \
118 ppc_enable_pmcs(); \ 118 EXTRA; \
119 mtspr(ADDRESS, *(unsigned long *)val); \ 119 mtspr(ADDRESS, *(unsigned long *)val); \
120} \ 120} \
121static ssize_t show_##NAME(struct device *dev, \ 121static ssize_t show_##NAME(struct device *dev, \
@@ -140,6 +140,10 @@ static ssize_t __used \
140 return count; \ 140 return count; \
141} 141}
142 142
143#define SYSFS_PMCSETUP(NAME, ADDRESS) \
144 __SYSFS_SPRSETUP(NAME, ADDRESS, ppc_enable_pmcs())
145#define SYSFS_SPRSETUP(NAME, ADDRESS) \
146 __SYSFS_SPRSETUP(NAME, ADDRESS, )
143 147
144/* Let's define all possible registers, we'll only hook up the ones 148/* Let's define all possible registers, we'll only hook up the ones
145 * that are implemented on the current processor 149 * that are implemented on the current processor
@@ -175,10 +179,10 @@ SYSFS_PMCSETUP(pmc7, SPRN_PMC7);
175SYSFS_PMCSETUP(pmc8, SPRN_PMC8); 179SYSFS_PMCSETUP(pmc8, SPRN_PMC8);
176 180
177SYSFS_PMCSETUP(mmcra, SPRN_MMCRA); 181SYSFS_PMCSETUP(mmcra, SPRN_MMCRA);
178SYSFS_PMCSETUP(purr, SPRN_PURR); 182SYSFS_SPRSETUP(purr, SPRN_PURR);
179SYSFS_PMCSETUP(spurr, SPRN_SPURR); 183SYSFS_SPRSETUP(spurr, SPRN_SPURR);
180SYSFS_PMCSETUP(dscr, SPRN_DSCR); 184SYSFS_SPRSETUP(dscr, SPRN_DSCR);
181SYSFS_PMCSETUP(pir, SPRN_PIR); 185SYSFS_SPRSETUP(pir, SPRN_PIR);
182 186
183/* 187/*
184 Lets only enable read for phyp resources and 188 Lets only enable read for phyp resources and
@@ -249,34 +253,34 @@ SYSFS_PMCSETUP(pa6t_pmc3, SPRN_PA6T_PMC3);
249SYSFS_PMCSETUP(pa6t_pmc4, SPRN_PA6T_PMC4); 253SYSFS_PMCSETUP(pa6t_pmc4, SPRN_PA6T_PMC4);
250SYSFS_PMCSETUP(pa6t_pmc5, SPRN_PA6T_PMC5); 254SYSFS_PMCSETUP(pa6t_pmc5, SPRN_PA6T_PMC5);
251#ifdef CONFIG_DEBUG_KERNEL 255#ifdef CONFIG_DEBUG_KERNEL
252SYSFS_PMCSETUP(hid0, SPRN_HID0); 256SYSFS_SPRSETUP(hid0, SPRN_HID0);
253SYSFS_PMCSETUP(hid1, SPRN_HID1); 257SYSFS_SPRSETUP(hid1, SPRN_HID1);
254SYSFS_PMCSETUP(hid4, SPRN_HID4); 258SYSFS_SPRSETUP(hid4, SPRN_HID4);
255SYSFS_PMCSETUP(hid5, SPRN_HID5); 259SYSFS_SPRSETUP(hid5, SPRN_HID5);
256SYSFS_PMCSETUP(ima0, SPRN_PA6T_IMA0); 260SYSFS_SPRSETUP(ima0, SPRN_PA6T_IMA0);
257SYSFS_PMCSETUP(ima1, SPRN_PA6T_IMA1); 261SYSFS_SPRSETUP(ima1, SPRN_PA6T_IMA1);
258SYSFS_PMCSETUP(ima2, SPRN_PA6T_IMA2); 262SYSFS_SPRSETUP(ima2, SPRN_PA6T_IMA2);
259SYSFS_PMCSETUP(ima3, SPRN_PA6T_IMA3); 263SYSFS_SPRSETUP(ima3, SPRN_PA6T_IMA3);
260SYSFS_PMCSETUP(ima4, SPRN_PA6T_IMA4); 264SYSFS_SPRSETUP(ima4, SPRN_PA6T_IMA4);
261SYSFS_PMCSETUP(ima5, SPRN_PA6T_IMA5); 265SYSFS_SPRSETUP(ima5, SPRN_PA6T_IMA5);
262SYSFS_PMCSETUP(ima6, SPRN_PA6T_IMA6); 266SYSFS_SPRSETUP(ima6, SPRN_PA6T_IMA6);
263SYSFS_PMCSETUP(ima7, SPRN_PA6T_IMA7); 267SYSFS_SPRSETUP(ima7, SPRN_PA6T_IMA7);
264SYSFS_PMCSETUP(ima8, SPRN_PA6T_IMA8); 268SYSFS_SPRSETUP(ima8, SPRN_PA6T_IMA8);
265SYSFS_PMCSETUP(ima9, SPRN_PA6T_IMA9); 269SYSFS_SPRSETUP(ima9, SPRN_PA6T_IMA9);
266SYSFS_PMCSETUP(imaat, SPRN_PA6T_IMAAT); 270SYSFS_SPRSETUP(imaat, SPRN_PA6T_IMAAT);
267SYSFS_PMCSETUP(btcr, SPRN_PA6T_BTCR); 271SYSFS_SPRSETUP(btcr, SPRN_PA6T_BTCR);
268SYSFS_PMCSETUP(pccr, SPRN_PA6T_PCCR); 272SYSFS_SPRSETUP(pccr, SPRN_PA6T_PCCR);
269SYSFS_PMCSETUP(rpccr, SPRN_PA6T_RPCCR); 273SYSFS_SPRSETUP(rpccr, SPRN_PA6T_RPCCR);
270SYSFS_PMCSETUP(der, SPRN_PA6T_DER); 274SYSFS_SPRSETUP(der, SPRN_PA6T_DER);
271SYSFS_PMCSETUP(mer, SPRN_PA6T_MER); 275SYSFS_SPRSETUP(mer, SPRN_PA6T_MER);
272SYSFS_PMCSETUP(ber, SPRN_PA6T_BER); 276SYSFS_SPRSETUP(ber, SPRN_PA6T_BER);
273SYSFS_PMCSETUP(ier, SPRN_PA6T_IER); 277SYSFS_SPRSETUP(ier, SPRN_PA6T_IER);
274SYSFS_PMCSETUP(sier, SPRN_PA6T_SIER); 278SYSFS_SPRSETUP(sier, SPRN_PA6T_SIER);
275SYSFS_PMCSETUP(siar, SPRN_PA6T_SIAR); 279SYSFS_SPRSETUP(siar, SPRN_PA6T_SIAR);
276SYSFS_PMCSETUP(tsr0, SPRN_PA6T_TSR0); 280SYSFS_SPRSETUP(tsr0, SPRN_PA6T_TSR0);
277SYSFS_PMCSETUP(tsr1, SPRN_PA6T_TSR1); 281SYSFS_SPRSETUP(tsr1, SPRN_PA6T_TSR1);
278SYSFS_PMCSETUP(tsr2, SPRN_PA6T_TSR2); 282SYSFS_SPRSETUP(tsr2, SPRN_PA6T_TSR2);
279SYSFS_PMCSETUP(tsr3, SPRN_PA6T_TSR3); 283SYSFS_SPRSETUP(tsr3, SPRN_PA6T_TSR3);
280#endif /* CONFIG_DEBUG_KERNEL */ 284#endif /* CONFIG_DEBUG_KERNEL */
281#endif /* HAS_PPC_PMC_PA6T */ 285#endif /* HAS_PPC_PMC_PA6T */
282 286
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index b3b144121cc9..afb1b56ef4fa 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -510,7 +510,6 @@ void timer_interrupt(struct pt_regs * regs)
510 */ 510 */
511 may_hard_irq_enable(); 511 may_hard_irq_enable();
512 512
513 __get_cpu_var(irq_stat).timer_irqs++;
514 513
515#if defined(CONFIG_PPC32) && defined(CONFIG_PMAC) 514#if defined(CONFIG_PPC32) && defined(CONFIG_PMAC)
516 if (atomic_read(&ppc_n_lost_interrupts) != 0) 515 if (atomic_read(&ppc_n_lost_interrupts) != 0)
@@ -532,10 +531,12 @@ void timer_interrupt(struct pt_regs * regs)
532 *next_tb = ~(u64)0; 531 *next_tb = ~(u64)0;
533 if (evt->event_handler) 532 if (evt->event_handler)
534 evt->event_handler(evt); 533 evt->event_handler(evt);
534 __get_cpu_var(irq_stat).timer_irqs_event++;
535 } else { 535 } else {
536 now = *next_tb - now; 536 now = *next_tb - now;
537 if (now <= DECREMENTER_MAX) 537 if (now <= DECREMENTER_MAX)
538 set_dec((int)now); 538 set_dec((int)now);
539 __get_cpu_var(irq_stat).timer_irqs_others++;
539 } 540 }
540 541
541#ifdef CONFIG_PPC64 542#ifdef CONFIG_PPC64
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 907a472f9a9e..330841766b09 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -285,6 +285,21 @@ void system_reset_exception(struct pt_regs *regs)
285 285
286 /* What should we do here? We could issue a shutdown or hard reset. */ 286 /* What should we do here? We could issue a shutdown or hard reset. */
287} 287}
288
289/*
290 * This function is called in real mode. Strictly no printk's please.
291 *
292 * regs->nip and regs->msr contains srr0 and ssr1.
293 */
294long machine_check_early(struct pt_regs *regs)
295{
296 long handled = 0;
297
298 if (cur_cpu_spec && cur_cpu_spec->machine_check_early)
299 handled = cur_cpu_spec->machine_check_early(regs);
300 return handled;
301}
302
288#endif 303#endif
289 304
290/* 305/*
diff --git a/arch/powerpc/kernel/vio.c b/arch/powerpc/kernel/vio.c
index 76a64821f4a2..826d8bd9e522 100644
--- a/arch/powerpc/kernel/vio.c
+++ b/arch/powerpc/kernel/vio.c
@@ -518,16 +518,18 @@ static dma_addr_t vio_dma_iommu_map_page(struct device *dev, struct page *page,
518 struct dma_attrs *attrs) 518 struct dma_attrs *attrs)
519{ 519{
520 struct vio_dev *viodev = to_vio_dev(dev); 520 struct vio_dev *viodev = to_vio_dev(dev);
521 struct iommu_table *tbl;
521 dma_addr_t ret = DMA_ERROR_CODE; 522 dma_addr_t ret = DMA_ERROR_CODE;
522 523
523 if (vio_cmo_alloc(viodev, roundup(size, IOMMU_PAGE_SIZE))) { 524 tbl = get_iommu_table_base(dev);
525 if (vio_cmo_alloc(viodev, roundup(size, IOMMU_PAGE_SIZE(tbl)))) {
524 atomic_inc(&viodev->cmo.allocs_failed); 526 atomic_inc(&viodev->cmo.allocs_failed);
525 return ret; 527 return ret;
526 } 528 }
527 529
528 ret = dma_iommu_ops.map_page(dev, page, offset, size, direction, attrs); 530 ret = dma_iommu_ops.map_page(dev, page, offset, size, direction, attrs);
529 if (unlikely(dma_mapping_error(dev, ret))) { 531 if (unlikely(dma_mapping_error(dev, ret))) {
530 vio_cmo_dealloc(viodev, roundup(size, IOMMU_PAGE_SIZE)); 532 vio_cmo_dealloc(viodev, roundup(size, IOMMU_PAGE_SIZE(tbl)));
531 atomic_inc(&viodev->cmo.allocs_failed); 533 atomic_inc(&viodev->cmo.allocs_failed);
532 } 534 }
533 535
@@ -540,10 +542,12 @@ static void vio_dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle,
540 struct dma_attrs *attrs) 542 struct dma_attrs *attrs)
541{ 543{
542 struct vio_dev *viodev = to_vio_dev(dev); 544 struct vio_dev *viodev = to_vio_dev(dev);
545 struct iommu_table *tbl;
543 546
547 tbl = get_iommu_table_base(dev);
544 dma_iommu_ops.unmap_page(dev, dma_handle, size, direction, attrs); 548 dma_iommu_ops.unmap_page(dev, dma_handle, size, direction, attrs);
545 549
546 vio_cmo_dealloc(viodev, roundup(size, IOMMU_PAGE_SIZE)); 550 vio_cmo_dealloc(viodev, roundup(size, IOMMU_PAGE_SIZE(tbl)));
547} 551}
548 552
549static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, 553static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -551,12 +555,14 @@ static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist,
551 struct dma_attrs *attrs) 555 struct dma_attrs *attrs)
552{ 556{
553 struct vio_dev *viodev = to_vio_dev(dev); 557 struct vio_dev *viodev = to_vio_dev(dev);
558 struct iommu_table *tbl;
554 struct scatterlist *sgl; 559 struct scatterlist *sgl;
555 int ret, count = 0; 560 int ret, count = 0;
556 size_t alloc_size = 0; 561 size_t alloc_size = 0;
557 562
563 tbl = get_iommu_table_base(dev);
558 for (sgl = sglist; count < nelems; count++, sgl++) 564 for (sgl = sglist; count < nelems; count++, sgl++)
559 alloc_size += roundup(sgl->length, IOMMU_PAGE_SIZE); 565 alloc_size += roundup(sgl->length, IOMMU_PAGE_SIZE(tbl));
560 566
561 if (vio_cmo_alloc(viodev, alloc_size)) { 567 if (vio_cmo_alloc(viodev, alloc_size)) {
562 atomic_inc(&viodev->cmo.allocs_failed); 568 atomic_inc(&viodev->cmo.allocs_failed);
@@ -572,7 +578,7 @@ static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist,
572 } 578 }
573 579
574 for (sgl = sglist, count = 0; count < ret; count++, sgl++) 580 for (sgl = sglist, count = 0; count < ret; count++, sgl++)
575 alloc_size -= roundup(sgl->dma_length, IOMMU_PAGE_SIZE); 581 alloc_size -= roundup(sgl->dma_length, IOMMU_PAGE_SIZE(tbl));
576 if (alloc_size) 582 if (alloc_size)
577 vio_cmo_dealloc(viodev, alloc_size); 583 vio_cmo_dealloc(viodev, alloc_size);
578 584
@@ -585,12 +591,14 @@ static void vio_dma_iommu_unmap_sg(struct device *dev,
585 struct dma_attrs *attrs) 591 struct dma_attrs *attrs)
586{ 592{
587 struct vio_dev *viodev = to_vio_dev(dev); 593 struct vio_dev *viodev = to_vio_dev(dev);
594 struct iommu_table *tbl;
588 struct scatterlist *sgl; 595 struct scatterlist *sgl;
589 size_t alloc_size = 0; 596 size_t alloc_size = 0;
590 int count = 0; 597 int count = 0;
591 598
599 tbl = get_iommu_table_base(dev);
592 for (sgl = sglist; count < nelems; count++, sgl++) 600 for (sgl = sglist; count < nelems; count++, sgl++)
593 alloc_size += roundup(sgl->dma_length, IOMMU_PAGE_SIZE); 601 alloc_size += roundup(sgl->dma_length, IOMMU_PAGE_SIZE(tbl));
594 602
595 dma_iommu_ops.unmap_sg(dev, sglist, nelems, direction, attrs); 603 dma_iommu_ops.unmap_sg(dev, sglist, nelems, direction, attrs);
596 604
@@ -706,11 +714,14 @@ static int vio_cmo_bus_probe(struct vio_dev *viodev)
706{ 714{
707 struct vio_cmo_dev_entry *dev_ent; 715 struct vio_cmo_dev_entry *dev_ent;
708 struct device *dev = &viodev->dev; 716 struct device *dev = &viodev->dev;
717 struct iommu_table *tbl;
709 struct vio_driver *viodrv = to_vio_driver(dev->driver); 718 struct vio_driver *viodrv = to_vio_driver(dev->driver);
710 unsigned long flags; 719 unsigned long flags;
711 size_t size; 720 size_t size;
712 bool dma_capable = false; 721 bool dma_capable = false;
713 722
723 tbl = get_iommu_table_base(dev);
724
714 /* A device requires entitlement if it has a DMA window property */ 725 /* A device requires entitlement if it has a DMA window property */
715 switch (viodev->family) { 726 switch (viodev->family) {
716 case VDEVICE: 727 case VDEVICE:
@@ -736,7 +747,8 @@ static int vio_cmo_bus_probe(struct vio_dev *viodev)
736 return -EINVAL; 747 return -EINVAL;
737 } 748 }
738 749
739 viodev->cmo.desired = IOMMU_PAGE_ALIGN(viodrv->get_desired_dma(viodev)); 750 viodev->cmo.desired =
751 IOMMU_PAGE_ALIGN(viodrv->get_desired_dma(viodev), tbl);
740 if (viodev->cmo.desired < VIO_CMO_MIN_ENT) 752 if (viodev->cmo.desired < VIO_CMO_MIN_ENT)
741 viodev->cmo.desired = VIO_CMO_MIN_ENT; 753 viodev->cmo.desired = VIO_CMO_MIN_ENT;
742 size = VIO_CMO_MIN_ENT; 754 size = VIO_CMO_MIN_ENT;
@@ -1176,9 +1188,10 @@ static struct iommu_table *vio_build_iommu_table(struct vio_dev *dev)
1176 &tbl->it_index, &offset, &size); 1188 &tbl->it_index, &offset, &size);
1177 1189
1178 /* TCE table size - measured in tce entries */ 1190 /* TCE table size - measured in tce entries */
1179 tbl->it_size = size >> IOMMU_PAGE_SHIFT; 1191 tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K;
1192 tbl->it_size = size >> tbl->it_page_shift;
1180 /* offset for VIO should always be 0 */ 1193 /* offset for VIO should always be 0 */
1181 tbl->it_offset = offset >> IOMMU_PAGE_SHIFT; 1194 tbl->it_offset = offset >> tbl->it_page_shift;
1182 tbl->it_busno = 0; 1195 tbl->it_busno = 0;
1183 tbl->it_type = TCE_VB; 1196 tbl->it_type = TCE_VB;
1184 tbl->it_blocksize = 16; 1197 tbl->it_blocksize = 16;
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-07 21:11:35 -0400