aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-06-02 17:17:38 -0400
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-06-09 02:47:37 -0400
commit0ebc4cdaa3fd7c9144d15fe9a6bcfcd310c265b8 (patch)
tree10b3bbb814415ef03d03cd674427a855967582b3 /arch
parent5b7c3c918c9c26c50d220b2b50359208cb5a1dbe (diff)
powerpc: Split exception handling out of head_64.S
To prepare for future support of Book3E 64-bit PowerPC processors, which use a completely different exception handling, we move that code to a new exceptions-64s.S file. This file is #included from head_64.S due to some of the absolute address requirements which can currently only be fulfilled from within that file. Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/powerpc/kernel/exceptions-64s.S978
-rw-r--r--arch/powerpc/kernel/head_64.S977
2 files changed, 995 insertions, 960 deletions
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
new file mode 100644
index 00000000000..eb898112e57
--- /dev/null
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -0,0 +1,978 @@
1/*
2 * This file contains the 64-bit "server" PowerPC variant
3 * of the low level exception handling including exception
4 * vectors, exception return, part of the slb and stab
5 * handling and other fixed offset specific things.
6 *
7 * This file is meant to be #included from head_64.S due to
8 * position dependant assembly.
9 *
10 * Most of this originates from head_64.S and thus has the same
11 * copyright history.
12 *
13 */
14
15/*
16 * We layout physical memory as follows:
17 * 0x0000 - 0x00ff : Secondary processor spin code
18 * 0x0100 - 0x2fff : pSeries Interrupt prologs
19 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
20 * 0x6000 - 0x6fff : Initial (CPU0) segment table
21 * 0x7000 - 0x7fff : FWNMI data area
22 * 0x8000 - : Early init and support code
23 */
24
25
26/*
27 * SPRG Usage
28 *
29 * Register Definition
30 *
31 * SPRG0 reserved for hypervisor
32 * SPRG1 temp - used to save gpr
33 * SPRG2 temp - used to save gpr
34 * SPRG3 virt addr of paca
35 */
36
37/*
38 * This is the start of the interrupt handlers for pSeries
39 * This code runs with relocation off.
40 * Code from here to __end_interrupts gets copied down to real
41 * address 0x100 when we are running a relocatable kernel.
42 * Therefore any relative branches in this section must only
43 * branch to labels in this section.
44 */
45 . = 0x100
46 .globl __start_interrupts
47__start_interrupts:
48
49 STD_EXCEPTION_PSERIES(0x100, system_reset)
50
51 . = 0x200
52_machine_check_pSeries:
53 HMT_MEDIUM
54 mtspr SPRN_SPRG1,r13 /* save r13 */
55 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
56
57 . = 0x300
58 .globl data_access_pSeries
59data_access_pSeries:
60 HMT_MEDIUM
61 mtspr SPRN_SPRG1,r13
62BEGIN_FTR_SECTION
63 mtspr SPRN_SPRG2,r12
64 mfspr r13,SPRN_DAR
65 mfspr r12,SPRN_DSISR
66 srdi r13,r13,60
67 rlwimi r13,r12,16,0x20
68 mfcr r12
69 cmpwi r13,0x2c
70 beq do_stab_bolted_pSeries
71 mtcrf 0x80,r12
72 mfspr r12,SPRN_SPRG2
73END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
74 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
75
76 . = 0x380
77 .globl data_access_slb_pSeries
78data_access_slb_pSeries:
79 HMT_MEDIUM
80 mtspr SPRN_SPRG1,r13
81 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
82 std r3,PACA_EXSLB+EX_R3(r13)
83 mfspr r3,SPRN_DAR
84 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
85 mfcr r9
86#ifdef __DISABLED__
87 /* Keep that around for when we re-implement dynamic VSIDs */
88 cmpdi r3,0
89 bge slb_miss_user_pseries
90#endif /* __DISABLED__ */
91 std r10,PACA_EXSLB+EX_R10(r13)
92 std r11,PACA_EXSLB+EX_R11(r13)
93 std r12,PACA_EXSLB+EX_R12(r13)
94 mfspr r10,SPRN_SPRG1
95 std r10,PACA_EXSLB+EX_R13(r13)
96 mfspr r12,SPRN_SRR1 /* and SRR1 */
97#ifndef CONFIG_RELOCATABLE
98 b .slb_miss_realmode
99#else
100 /*
101 * We can't just use a direct branch to .slb_miss_realmode
102 * because the distance from here to there depends on where
103 * the kernel ends up being put.
104 */
105 mfctr r11
106 ld r10,PACAKBASE(r13)
107 LOAD_HANDLER(r10, .slb_miss_realmode)
108 mtctr r10
109 bctr
110#endif
111
112 STD_EXCEPTION_PSERIES(0x400, instruction_access)
113
114 . = 0x480
115 .globl instruction_access_slb_pSeries
116instruction_access_slb_pSeries:
117 HMT_MEDIUM
118 mtspr SPRN_SPRG1,r13
119 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
120 std r3,PACA_EXSLB+EX_R3(r13)
121 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
122 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
123 mfcr r9
124#ifdef __DISABLED__
125 /* Keep that around for when we re-implement dynamic VSIDs */
126 cmpdi r3,0
127 bge slb_miss_user_pseries
128#endif /* __DISABLED__ */
129 std r10,PACA_EXSLB+EX_R10(r13)
130 std r11,PACA_EXSLB+EX_R11(r13)
131 std r12,PACA_EXSLB+EX_R12(r13)
132 mfspr r10,SPRN_SPRG1
133 std r10,PACA_EXSLB+EX_R13(r13)
134 mfspr r12,SPRN_SRR1 /* and SRR1 */
135#ifndef CONFIG_RELOCATABLE
136 b .slb_miss_realmode
137#else
138 mfctr r11
139 ld r10,PACAKBASE(r13)
140 LOAD_HANDLER(r10, .slb_miss_realmode)
141 mtctr r10
142 bctr
143#endif
144
145 MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
146 STD_EXCEPTION_PSERIES(0x600, alignment)
147 STD_EXCEPTION_PSERIES(0x700, program_check)
148 STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
149 MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
150 STD_EXCEPTION_PSERIES(0xa00, trap_0a)
151 STD_EXCEPTION_PSERIES(0xb00, trap_0b)
152
153 . = 0xc00
154 .globl system_call_pSeries
155system_call_pSeries:
156 HMT_MEDIUM
157BEGIN_FTR_SECTION
158 cmpdi r0,0x1ebe
159 beq- 1f
160END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
161 mr r9,r13
162 mfspr r13,SPRN_SPRG3
163 mfspr r11,SPRN_SRR0
164 ld r12,PACAKBASE(r13)
165 ld r10,PACAKMSR(r13)
166 LOAD_HANDLER(r12, system_call_entry)
167 mtspr SPRN_SRR0,r12
168 mfspr r12,SPRN_SRR1
169 mtspr SPRN_SRR1,r10
170 rfid
171 b . /* prevent speculative execution */
172
173/* Fast LE/BE switch system call */
1741: mfspr r12,SPRN_SRR1
175 xori r12,r12,MSR_LE
176 mtspr SPRN_SRR1,r12
177 rfid /* return to userspace */
178 b .
179
180 STD_EXCEPTION_PSERIES(0xd00, single_step)
181 STD_EXCEPTION_PSERIES(0xe00, trap_0e)
182
183 /* We need to deal with the Altivec unavailable exception
184 * here which is at 0xf20, thus in the middle of the
185 * prolog code of the PerformanceMonitor one. A little
186 * trickery is thus necessary
187 */
188 . = 0xf00
189 b performance_monitor_pSeries
190
191 . = 0xf20
192 b altivec_unavailable_pSeries
193
194 . = 0xf40
195 b vsx_unavailable_pSeries
196
197#ifdef CONFIG_CBE_RAS
198 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
199#endif /* CONFIG_CBE_RAS */
200 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
201#ifdef CONFIG_CBE_RAS
202 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
203#endif /* CONFIG_CBE_RAS */
204 STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
205#ifdef CONFIG_CBE_RAS
206 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
207#endif /* CONFIG_CBE_RAS */
208
209 . = 0x3000
210
211/*** pSeries interrupt support ***/
212
213 /* moved from 0xf00 */
214 STD_EXCEPTION_PSERIES(., performance_monitor)
215 STD_EXCEPTION_PSERIES(., altivec_unavailable)
216 STD_EXCEPTION_PSERIES(., vsx_unavailable)
217
218/*
219 * An interrupt came in while soft-disabled; clear EE in SRR1,
220 * clear paca->hard_enabled and return.
221 */
222masked_interrupt:
223 stb r10,PACAHARDIRQEN(r13)
224 mtcrf 0x80,r9
225 ld r9,PACA_EXGEN+EX_R9(r13)
226 mfspr r10,SPRN_SRR1
227 rldicl r10,r10,48,1 /* clear MSR_EE */
228 rotldi r10,r10,16
229 mtspr SPRN_SRR1,r10
230 ld r10,PACA_EXGEN+EX_R10(r13)
231 mfspr r13,SPRN_SPRG1
232 rfid
233 b .
234
235 .align 7
236do_stab_bolted_pSeries:
237 mtcrf 0x80,r12
238 mfspr r12,SPRN_SPRG2
239 EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
240
241#ifdef CONFIG_PPC_PSERIES
242/*
243 * Vectors for the FWNMI option. Share common code.
244 */
245 .globl system_reset_fwnmi
246 .align 7
247system_reset_fwnmi:
248 HMT_MEDIUM
249 mtspr SPRN_SPRG1,r13 /* save r13 */
250 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
251
252 .globl machine_check_fwnmi
253 .align 7
254machine_check_fwnmi:
255 HMT_MEDIUM
256 mtspr SPRN_SPRG1,r13 /* save r13 */
257 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
258
259#endif /* CONFIG_PPC_PSERIES */
260
261#ifdef __DISABLED__
262/*
263 * This is used for when the SLB miss handler has to go virtual,
264 * which doesn't happen for now anymore but will once we re-implement
265 * dynamic VSIDs for shared page tables
266 */
267slb_miss_user_pseries:
268 std r10,PACA_EXGEN+EX_R10(r13)
269 std r11,PACA_EXGEN+EX_R11(r13)
270 std r12,PACA_EXGEN+EX_R12(r13)
271 mfspr r10,SPRG1
272 ld r11,PACA_EXSLB+EX_R9(r13)
273 ld r12,PACA_EXSLB+EX_R3(r13)
274 std r10,PACA_EXGEN+EX_R13(r13)
275 std r11,PACA_EXGEN+EX_R9(r13)
276 std r12,PACA_EXGEN+EX_R3(r13)
277 clrrdi r12,r13,32
278 mfmsr r10
279 mfspr r11,SRR0 /* save SRR0 */
280 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
281 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
282 mtspr SRR0,r12
283 mfspr r12,SRR1 /* and SRR1 */
284 mtspr SRR1,r10
285 rfid
286 b . /* prevent spec. execution */
287#endif /* __DISABLED__ */
288
289 .align 7
290 .globl __end_interrupts
291__end_interrupts:
292
293/*
294 * Code from here down to __end_handlers is invoked from the
295 * exception prologs above. Because the prologs assemble the
296 * addresses of these handlers using the LOAD_HANDLER macro,
297 * which uses an addi instruction, these handlers must be in
298 * the first 32k of the kernel image.
299 */
300
301/*** Common interrupt handlers ***/
302
303 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
304
305 /*
306 * Machine check is different because we use a different
307 * save area: PACA_EXMC instead of PACA_EXGEN.
308 */
309 .align 7
310 .globl machine_check_common
311machine_check_common:
312 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
313 FINISH_NAP
314 DISABLE_INTS
315 bl .save_nvgprs
316 addi r3,r1,STACK_FRAME_OVERHEAD
317 bl .machine_check_exception
318 b .ret_from_except
319
320 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
321 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
322 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
323 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
324 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
325 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
326 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
327#ifdef CONFIG_ALTIVEC
328 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
329#else
330 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
331#endif
332#ifdef CONFIG_CBE_RAS
333 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
334 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
335 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
336#endif /* CONFIG_CBE_RAS */
337
338 .align 7
339system_call_entry:
340 b system_call_common
341
342/*
343 * Here we have detected that the kernel stack pointer is bad.
344 * R9 contains the saved CR, r13 points to the paca,
345 * r10 contains the (bad) kernel stack pointer,
346 * r11 and r12 contain the saved SRR0 and SRR1.
347 * We switch to using an emergency stack, save the registers there,
348 * and call kernel_bad_stack(), which panics.
349 */
350bad_stack:
351 ld r1,PACAEMERGSP(r13)
352 subi r1,r1,64+INT_FRAME_SIZE
353 std r9,_CCR(r1)
354 std r10,GPR1(r1)
355 std r11,_NIP(r1)
356 std r12,_MSR(r1)
357 mfspr r11,SPRN_DAR
358 mfspr r12,SPRN_DSISR
359 std r11,_DAR(r1)
360 std r12,_DSISR(r1)
361 mflr r10
362 mfctr r11
363 mfxer r12
364 std r10,_LINK(r1)
365 std r11,_CTR(r1)
366 std r12,_XER(r1)
367 SAVE_GPR(0,r1)
368 SAVE_GPR(2,r1)
369 SAVE_4GPRS(3,r1)
370 SAVE_2GPRS(7,r1)
371 SAVE_10GPRS(12,r1)
372 SAVE_10GPRS(22,r1)
373 lhz r12,PACA_TRAP_SAVE(r13)
374 std r12,_TRAP(r1)
375 addi r11,r1,INT_FRAME_SIZE
376 std r11,0(r1)
377 li r12,0
378 std r12,0(r11)
379 ld r2,PACATOC(r13)
3801: addi r3,r1,STACK_FRAME_OVERHEAD
381 bl .kernel_bad_stack
382 b 1b
383
384/*
385 * Here r13 points to the paca, r9 contains the saved CR,
386 * SRR0 and SRR1 are saved in r11 and r12,
387 * r9 - r13 are saved in paca->exgen.
388 */
389 .align 7
390 .globl data_access_common
391data_access_common:
392 mfspr r10,SPRN_DAR
393 std r10,PACA_EXGEN+EX_DAR(r13)
394 mfspr r10,SPRN_DSISR
395 stw r10,PACA_EXGEN+EX_DSISR(r13)
396 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
397 ld r3,PACA_EXGEN+EX_DAR(r13)
398 lwz r4,PACA_EXGEN+EX_DSISR(r13)
399 li r5,0x300
400 b .do_hash_page /* Try to handle as hpte fault */
401
402 .align 7
403 .globl instruction_access_common
404instruction_access_common:
405 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
406 ld r3,_NIP(r1)
407 andis. r4,r12,0x5820
408 li r5,0x400
409 b .do_hash_page /* Try to handle as hpte fault */
410
411/*
412 * Here is the common SLB miss user that is used when going to virtual
413 * mode for SLB misses, that is currently not used
414 */
415#ifdef __DISABLED__
416 .align 7
417 .globl slb_miss_user_common
418slb_miss_user_common:
419 mflr r10
420 std r3,PACA_EXGEN+EX_DAR(r13)
421 stw r9,PACA_EXGEN+EX_CCR(r13)
422 std r10,PACA_EXGEN+EX_LR(r13)
423 std r11,PACA_EXGEN+EX_SRR0(r13)
424 bl .slb_allocate_user
425
426 ld r10,PACA_EXGEN+EX_LR(r13)
427 ld r3,PACA_EXGEN+EX_R3(r13)
428 lwz r9,PACA_EXGEN+EX_CCR(r13)
429 ld r11,PACA_EXGEN+EX_SRR0(r13)
430 mtlr r10
431 beq- slb_miss_fault
432
433 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
434 beq- unrecov_user_slb
435 mfmsr r10
436
437.machine push
438.machine "power4"
439 mtcrf 0x80,r9
440.machine pop
441
442 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
443 mtmsrd r10,1
444
445 mtspr SRR0,r11
446 mtspr SRR1,r12
447
448 ld r9,PACA_EXGEN+EX_R9(r13)
449 ld r10,PACA_EXGEN+EX_R10(r13)
450 ld r11,PACA_EXGEN+EX_R11(r13)
451 ld r12,PACA_EXGEN+EX_R12(r13)
452 ld r13,PACA_EXGEN+EX_R13(r13)
453 rfid
454 b .
455
456slb_miss_fault:
457 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
458 ld r4,PACA_EXGEN+EX_DAR(r13)
459 li r5,0
460 std r4,_DAR(r1)
461 std r5,_DSISR(r1)
462 b handle_page_fault
463
464unrecov_user_slb:
465 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
466 DISABLE_INTS
467 bl .save_nvgprs
4681: addi r3,r1,STACK_FRAME_OVERHEAD
469 bl .unrecoverable_exception
470 b 1b
471
472#endif /* __DISABLED__ */
473
474
475/*
476 * r13 points to the PACA, r9 contains the saved CR,
477 * r12 contain the saved SRR1, SRR0 is still ready for return
478 * r3 has the faulting address
479 * r9 - r13 are saved in paca->exslb.
480 * r3 is saved in paca->slb_r3
481 * We assume we aren't going to take any exceptions during this procedure.
482 */
483_GLOBAL(slb_miss_realmode)
484 mflr r10
485#ifdef CONFIG_RELOCATABLE
486 mtctr r11
487#endif
488
489 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
490 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
491
492 bl .slb_allocate_realmode
493
494 /* All done -- return from exception. */
495
496 ld r10,PACA_EXSLB+EX_LR(r13)
497 ld r3,PACA_EXSLB+EX_R3(r13)
498 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
499#ifdef CONFIG_PPC_ISERIES
500BEGIN_FW_FTR_SECTION
501 ld r11,PACALPPACAPTR(r13)
502 ld r11,LPPACASRR0(r11) /* get SRR0 value */
503END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
504#endif /* CONFIG_PPC_ISERIES */
505
506 mtlr r10
507
508 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
509 beq- 2f
510
511.machine push
512.machine "power4"
513 mtcrf 0x80,r9
514 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
515.machine pop
516
517#ifdef CONFIG_PPC_ISERIES
518BEGIN_FW_FTR_SECTION
519 mtspr SPRN_SRR0,r11
520 mtspr SPRN_SRR1,r12
521END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
522#endif /* CONFIG_PPC_ISERIES */
523 ld r9,PACA_EXSLB+EX_R9(r13)
524 ld r10,PACA_EXSLB+EX_R10(r13)
525 ld r11,PACA_EXSLB+EX_R11(r13)
526 ld r12,PACA_EXSLB+EX_R12(r13)
527 ld r13,PACA_EXSLB+EX_R13(r13)
528 rfid
529 b . /* prevent speculative execution */
530
5312:
532#ifdef CONFIG_PPC_ISERIES
533BEGIN_FW_FTR_SECTION
534 b unrecov_slb
535END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
536#endif /* CONFIG_PPC_ISERIES */
537 mfspr r11,SPRN_SRR0
538 ld r10,PACAKBASE(r13)
539 LOAD_HANDLER(r10,unrecov_slb)
540 mtspr SPRN_SRR0,r10
541 ld r10,PACAKMSR(r13)
542 mtspr SPRN_SRR1,r10
543 rfid
544 b .
545
546unrecov_slb:
547 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
548 DISABLE_INTS
549 bl .save_nvgprs
5501: addi r3,r1,STACK_FRAME_OVERHEAD
551 bl .unrecoverable_exception
552 b 1b
553
554 .align 7
555 .globl hardware_interrupt_common
556 .globl hardware_interrupt_entry
557hardware_interrupt_common:
558 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
559 FINISH_NAP
560hardware_interrupt_entry:
561 DISABLE_INTS
562BEGIN_FTR_SECTION
563 bl .ppc64_runlatch_on
564END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
565 addi r3,r1,STACK_FRAME_OVERHEAD
566 bl .do_IRQ
567 b .ret_from_except_lite
568
569#ifdef CONFIG_PPC_970_NAP
570power4_fixup_nap:
571 andc r9,r9,r10
572 std r9,TI_LOCAL_FLAGS(r11)
573 ld r10,_LINK(r1) /* make idle task do the */
574 std r10,_NIP(r1) /* equivalent of a blr */
575 blr
576#endif
577
578 .align 7
579 .globl alignment_common
580alignment_common:
581 mfspr r10,SPRN_DAR
582 std r10,PACA_EXGEN+EX_DAR(r13)
583 mfspr r10,SPRN_DSISR
584 stw r10,PACA_EXGEN+EX_DSISR(r13)
585 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
586 ld r3,PACA_EXGEN+EX_DAR(r13)
587 lwz r4,PACA_EXGEN+EX_DSISR(r13)
588 std r3,_DAR(r1)
589 std r4,_DSISR(r1)
590 bl .save_nvgprs
591 addi r3,r1,STACK_FRAME_OVERHEAD
592 ENABLE_INTS
593 bl .alignment_exception
594 b .ret_from_except
595
596 .align 7
597 .globl program_check_common
598program_check_common:
599 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
600 bl .save_nvgprs
601 addi r3,r1,STACK_FRAME_OVERHEAD
602 ENABLE_INTS
603 bl .program_check_exception
604 b .ret_from_except
605
606 .align 7
607 .globl fp_unavailable_common
608fp_unavailable_common:
609 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
610 bne 1f /* if from user, just load it up */
611 bl .save_nvgprs
612 addi r3,r1,STACK_FRAME_OVERHEAD
613 ENABLE_INTS
614 bl .kernel_fp_unavailable_exception
615 BUG_OPCODE
6161: bl .load_up_fpu
617 b fast_exception_return
618
619 .align 7
620 .globl altivec_unavailable_common
621altivec_unavailable_common:
622 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
623#ifdef CONFIG_ALTIVEC
624BEGIN_FTR_SECTION
625 beq 1f
626 bl .load_up_altivec
627 b fast_exception_return
6281:
629END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
630#endif
631 bl .save_nvgprs
632 addi r3,r1,STACK_FRAME_OVERHEAD
633 ENABLE_INTS
634 bl .altivec_unavailable_exception
635 b .ret_from_except
636
637 .align 7
638 .globl vsx_unavailable_common
639vsx_unavailable_common:
640 EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
641#ifdef CONFIG_VSX
642BEGIN_FTR_SECTION
643 bne .load_up_vsx
6441:
645END_FTR_SECTION_IFSET(CPU_FTR_VSX)
646#endif
647 bl .save_nvgprs
648 addi r3,r1,STACK_FRAME_OVERHEAD
649 ENABLE_INTS
650 bl .vsx_unavailable_exception
651 b .ret_from_except
652
653 .align 7
654 .globl __end_handlers
655__end_handlers:
656
657/*
658 * Return from an exception with minimal checks.
659 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
660 * If interrupts have been enabled, or anything has been
661 * done that might have changed the scheduling status of
662 * any task or sent any task a signal, you should use
663 * ret_from_except or ret_from_except_lite instead of this.
664 */
665fast_exc_return_irq: /* restores irq state too */
666 ld r3,SOFTE(r1)
667 TRACE_AND_RESTORE_IRQ(r3);
668 ld r12,_MSR(r1)
669 rldicl r4,r12,49,63 /* get MSR_EE to LSB */
670 stb r4,PACAHARDIRQEN(r13) /* restore paca->hard_enabled */
671 b 1f
672
673 .globl fast_exception_return
674fast_exception_return:
675 ld r12,_MSR(r1)
6761: ld r11,_NIP(r1)
677 andi. r3,r12,MSR_RI /* check if RI is set */
678 beq- unrecov_fer
679
680#ifdef CONFIG_VIRT_CPU_ACCOUNTING
681 andi. r3,r12,MSR_PR
682 beq 2f
683 ACCOUNT_CPU_USER_EXIT(r3, r4)
6842:
685#endif
686
687 ld r3,_CCR(r1)
688 ld r4,_LINK(r1)
689 ld r5,_CTR(r1)
690 ld r6,_XER(r1)
691 mtcr r3
692 mtlr r4
693 mtctr r5
694 mtxer r6
695 REST_GPR(0, r1)
696 REST_8GPRS(2, r1)
697
698 mfmsr r10
699 rldicl r10,r10,48,1 /* clear EE */
700 rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */
701 mtmsrd r10,1
702
703 mtspr SPRN_SRR1,r12
704 mtspr SPRN_SRR0,r11
705 REST_4GPRS(10, r1)
706 ld r1,GPR1(r1)
707 rfid
708 b . /* prevent speculative execution */
709
710unrecov_fer:
711 bl .save_nvgprs
7121: addi r3,r1,STACK_FRAME_OVERHEAD
713 bl .unrecoverable_exception
714 b 1b
715
716
717/*
718 * Hash table stuff
719 */
720 .align 7
721_STATIC(do_hash_page)
722 std r3,_DAR(r1)
723 std r4,_DSISR(r1)
724
725 andis. r0,r4,0xa450 /* weird error? */
726 bne- handle_page_fault /* if not, try to insert a HPTE */
727BEGIN_FTR_SECTION
728 andis. r0,r4,0x0020 /* Is it a segment table fault? */
729 bne- do_ste_alloc /* If so handle it */
730END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
731
732 /*
733 * On iSeries, we soft-disable interrupts here, then
734 * hard-enable interrupts so that the hash_page code can spin on
735 * the hash_table_lock without problems on a shared processor.
736 */
737 DISABLE_INTS
738
739 /*
740 * Currently, trace_hardirqs_off() will be called by DISABLE_INTS
741 * and will clobber volatile registers when irq tracing is enabled
742 * so we need to reload them. It may be possible to be smarter here
743 * and move the irq tracing elsewhere but let's keep it simple for
744 * now
745 */
746#ifdef CONFIG_TRACE_IRQFLAGS
747 ld r3,_DAR(r1)
748 ld r4,_DSISR(r1)
749 ld r5,_TRAP(r1)
750 ld r12,_MSR(r1)
751 clrrdi r5,r5,4
752#endif /* CONFIG_TRACE_IRQFLAGS */
753 /*
754 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
755 * accessing a userspace segment (even from the kernel). We assume
756 * kernel addresses always have the high bit set.
757 */
758 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
759 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
760 orc r0,r12,r0 /* MSR_PR | ~high_bit */
761 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
762 ori r4,r4,1 /* add _PAGE_PRESENT */
763 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
764
765 /*
766 * r3 contains the faulting address
767 * r4 contains the required access permissions
768 * r5 contains the trap number
769 *
770 * at return r3 = 0 for success
771 */
772 bl .hash_page /* build HPTE if possible */
773 cmpdi r3,0 /* see if hash_page succeeded */
774
775BEGIN_FW_FTR_SECTION
776 /*
777 * If we had interrupts soft-enabled at the point where the
778 * DSI/ISI occurred, and an interrupt came in during hash_page,
779 * handle it now.
780 * We jump to ret_from_except_lite rather than fast_exception_return
781 * because ret_from_except_lite will check for and handle pending
782 * interrupts if necessary.
783 */
784 beq 13f
785END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
786
787BEGIN_FW_FTR_SECTION
788 /*
789 * Here we have interrupts hard-disabled, so it is sufficient
790 * to restore paca->{soft,hard}_enable and get out.
791 */
792 beq fast_exc_return_irq /* Return from exception on success */
793END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
794
795 /* For a hash failure, we don't bother re-enabling interrupts */
796 ble- 12f
797
798 /*
799 * hash_page couldn't handle it, set soft interrupt enable back
800 * to what it was before the trap. Note that .raw_local_irq_restore
801 * handles any interrupts pending at this point.
802 */
803 ld r3,SOFTE(r1)
804 TRACE_AND_RESTORE_IRQ_PARTIAL(r3, 11f)
805 bl .raw_local_irq_restore
806 b 11f
807
808/* Here we have a page fault that hash_page can't handle. */
809handle_page_fault:
810 ENABLE_INTS
81111: ld r4,_DAR(r1)
812 ld r5,_DSISR(r1)
813 addi r3,r1,STACK_FRAME_OVERHEAD
814 bl .do_page_fault
815 cmpdi r3,0
816 beq+ 13f
817 bl .save_nvgprs
818 mr r5,r3
819 addi r3,r1,STACK_FRAME_OVERHEAD
820 lwz r4,_DAR(r1)
821 bl .bad_page_fault
822 b .ret_from_except
823
82413: b .ret_from_except_lite
825
826/* We have a page fault that hash_page could handle but HV refused
827 * the PTE insertion
828 */
82912: bl .save_nvgprs
830 mr r5,r3
831 addi r3,r1,STACK_FRAME_OVERHEAD
832 ld r4,_DAR(r1)
833 bl .low_hash_fault
834 b .ret_from_except
835
836 /* here we have a segment miss */
837do_ste_alloc:
838 bl .ste_allocate /* try to insert stab entry */
839 cmpdi r3,0
840 bne- handle_page_fault
841 b fast_exception_return
842
843/*
844 * r13 points to the PACA, r9 contains the saved CR,
845 * r11 and r12 contain the saved SRR0 and SRR1.
846 * r9 - r13 are saved in paca->exslb.
847 * We assume we aren't going to take any exceptions during this procedure.
848 * We assume (DAR >> 60) == 0xc.
849 */
850 .align 7
851_GLOBAL(do_stab_bolted)
852 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
853 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
854
855 /* Hash to the primary group */
856 ld r10,PACASTABVIRT(r13)
857 mfspr r11,SPRN_DAR
858 srdi r11,r11,28
859 rldimi r10,r11,7,52 /* r10 = first ste of the group */
860
861 /* Calculate VSID */
862 /* This is a kernel address, so protovsid = ESID */
863 ASM_VSID_SCRAMBLE(r11, r9, 256M)
864 rldic r9,r11,12,16 /* r9 = vsid << 12 */
865
866 /* Search the primary group for a free entry */
8671: ld r11,0(r10) /* Test valid bit of the current ste */
868 andi. r11,r11,0x80
869 beq 2f
870 addi r10,r10,16
871 andi. r11,r10,0x70
872 bne 1b
873
874 /* Stick for only searching the primary group for now. */
875 /* At least for now, we use a very simple random castout scheme */
876 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */
877 mftb r11
878 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */
879 ori r11,r11,0x10
880
881 /* r10 currently points to an ste one past the group of interest */
882 /* make it point to the randomly selected entry */
883 subi r10,r10,128
884 or r10,r10,r11 /* r10 is the entry to invalidate */
885
886 isync /* mark the entry invalid */
887 ld r11,0(r10)
888 rldicl r11,r11,56,1 /* clear the valid bit */
889 rotldi r11,r11,8
890 std r11,0(r10)
891 sync
892
893 clrrdi r11,r11,28 /* Get the esid part of the ste */
894 slbie r11
895
8962: std r9,8(r10) /* Store the vsid part of the ste */
897 eieio
898
899 mfspr r11,SPRN_DAR /* Get the new esid */
900 clrrdi r11,r11,28 /* Permits a full 32b of ESID */
901 ori r11,r11,0x90 /* Turn on valid and kp */
902 std r11,0(r10) /* Put new entry back into the stab */
903
904 sync
905
906 /* All done -- return from exception. */
907 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
908 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
909
910 andi. r10,r12,MSR_RI
911 beq- unrecov_slb
912
913 mtcrf 0x80,r9 /* restore CR */
914
915 mfmsr r10
916 clrrdi r10,r10,2
917 mtmsrd r10,1
918
919 mtspr SPRN_SRR0,r11
920 mtspr SPRN_SRR1,r12
921 ld r9,PACA_EXSLB+EX_R9(r13)
922 ld r10,PACA_EXSLB+EX_R10(r13)
923 ld r11,PACA_EXSLB+EX_R11(r13)
924 ld r12,PACA_EXSLB+EX_R12(r13)
925 ld r13,PACA_EXSLB+EX_R13(r13)
926 rfid
927 b . /* prevent speculative execution */
928
929/*
930 * Space for CPU0's segment table.
931 *
932 * On iSeries, the hypervisor must fill in at least one entry before
933 * we get control (with relocate on). The address is given to the hv
934 * as a page number (see xLparMap below), so this must be at a
935 * fixed address (the linker can't compute (u64)&initial_stab >>
936 * PAGE_SHIFT).
937 */
938 . = STAB0_OFFSET /* 0x6000 */
939 .globl initial_stab
940initial_stab:
941 .space 4096
942
943#ifdef CONFIG_PPC_PSERIES
944/*
945 * Data area reserved for FWNMI option.
946 * This address (0x7000) is fixed by the RPA.
947 */
948 .= 0x7000
949 .globl fwnmi_data_area
950fwnmi_data_area:
951#endif /* CONFIG_PPC_PSERIES */
952
953 /* iSeries does not use the FWNMI stuff, so it is safe to put
954 * this here, even if we later allow kernels that will boot on
955 * both pSeries and iSeries */
956#ifdef CONFIG_PPC_ISERIES
957 . = LPARMAP_PHYS
958 .globl xLparMap
959xLparMap:
960 .quad HvEsidsToMap /* xNumberEsids */
961 .quad HvRangesToMap /* xNumberRanges */
962 .quad STAB0_PAGE /* xSegmentTableOffs */
963 .zero 40 /* xRsvd */
964 /* xEsids (HvEsidsToMap entries of 2 quads) */
965 .quad PAGE_OFFSET_ESID /* xKernelEsid */
966 .quad PAGE_OFFSET_VSID /* xKernelVsid */
967 .quad VMALLOC_START_ESID /* xKernelEsid */
968 .quad VMALLOC_START_VSID /* xKernelVsid */
969 /* xRanges (HvRangesToMap entries of 3 quads) */
970 .quad HvPagesToMap /* xPages */
971 .quad 0 /* xOffset */
972 .quad PAGE_OFFSET_VSID << (SID_SHIFT - HW_PAGE_SHIFT) /* xVPN */
973
974#endif /* CONFIG_PPC_ISERIES */
975
976#ifdef CONFIG_PPC_PSERIES
977 . = 0x8000
978#endif /* CONFIG_PPC_PSERIES */
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 382495fa90b..012505ebd9f 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -12,8 +12,9 @@
12 * Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and 12 * Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
13 * Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com 13 * Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
14 * 14 *
15 * This file contains the low-level support and setup for the 15 * This file contains the entry point for the 64-bit kernel along
16 * PowerPC-64 platform, including trap and interrupt dispatch. 16 * with some early initialization code common to all 64-bit powerpc
17 * variants.
17 * 18 *
18 * This program is free software; you can redistribute it and/or 19 * This program is free software; you can redistribute it and/or
19 * modify it under the terms of the GNU General Public License 20 * modify it under the terms of the GNU General Public License
@@ -38,36 +39,25 @@
38#include <asm/exception.h> 39#include <asm/exception.h>
39#include <asm/irqflags.h> 40#include <asm/irqflags.h>
40 41
41/* 42/* The physical memory is layed out such that the secondary processor
42 * We layout physical memory as follows: 43 * spin code sits at 0x0000...0x00ff. On server, the vectors follow
43 * 0x0000 - 0x00ff : Secondary processor spin code 44 * using the layout described in exceptions-64s.S
44 * 0x0100 - 0x2fff : pSeries Interrupt prologs
45 * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
46 * 0x6000 - 0x6fff : Initial (CPU0) segment table
47 * 0x7000 - 0x7fff : FWNMI data area
48 * 0x8000 - : Early init and support code
49 */
50
51/*
52 * SPRG Usage
53 *
54 * Register Definition
55 *
56 * SPRG0 reserved for hypervisor
57 * SPRG1 temp - used to save gpr
58 * SPRG2 temp - used to save gpr
59 * SPRG3 virt addr of paca
60 */ 45 */
61 46
62/* 47/*
63 * Entering into this code we make the following assumptions: 48 * Entering into this code we make the following assumptions:
64 * For pSeries: 49 *
50 * For pSeries or server processors:
65 * 1. The MMU is off & open firmware is running in real mode. 51 * 1. The MMU is off & open firmware is running in real mode.
66 * 2. The kernel is entered at __start 52 * 2. The kernel is entered at __start
67 * 53 *
68 * For iSeries: 54 * For iSeries:
69 * 1. The MMU is on (as it always is for iSeries) 55 * 1. The MMU is on (as it always is for iSeries)
70 * 2. The kernel is entered at system_reset_iSeries 56 * 2. The kernel is entered at system_reset_iSeries
57 *
58 * For Book3E processors:
59 * 1. The MMU is on running in AS0 in a state defined in ePAPR
60 * 2. The kernel is entered at __start
71 */ 61 */
72 62
73 .text 63 .text
@@ -166,947 +156,14 @@ exception_marker:
166 .text 156 .text
167 157
168/* 158/*
169 * This is the start of the interrupt handlers for pSeries 159 * On server, we include the exception vectors code here as it
170 * This code runs with relocation off. 160 * relies on absolute addressing which is only possible within
171 * Code from here to __end_interrupts gets copied down to real 161 * this compilation unit
172 * address 0x100 when we are running a relocatable kernel.
173 * Therefore any relative branches in this section must only
174 * branch to labels in this section.
175 */
176 . = 0x100
177 .globl __start_interrupts
178__start_interrupts:
179
180 STD_EXCEPTION_PSERIES(0x100, system_reset)
181
182 . = 0x200
183_machine_check_pSeries:
184 HMT_MEDIUM
185 mtspr SPRN_SPRG1,r13 /* save r13 */
186 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
187
188 . = 0x300
189 .globl data_access_pSeries
190data_access_pSeries:
191 HMT_MEDIUM
192 mtspr SPRN_SPRG1,r13
193BEGIN_FTR_SECTION
194 mtspr SPRN_SPRG2,r12
195 mfspr r13,SPRN_DAR
196 mfspr r12,SPRN_DSISR
197 srdi r13,r13,60
198 rlwimi r13,r12,16,0x20
199 mfcr r12
200 cmpwi r13,0x2c
201 beq do_stab_bolted_pSeries
202 mtcrf 0x80,r12
203 mfspr r12,SPRN_SPRG2
204END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
205 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
206
207 . = 0x380
208 .globl data_access_slb_pSeries
209data_access_slb_pSeries:
210 HMT_MEDIUM
211 mtspr SPRN_SPRG1,r13
212 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
213 std r3,PACA_EXSLB+EX_R3(r13)
214 mfspr r3,SPRN_DAR
215 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
216 mfcr r9
217#ifdef __DISABLED__
218 /* Keep that around for when we re-implement dynamic VSIDs */
219 cmpdi r3,0
220 bge slb_miss_user_pseries
221#endif /* __DISABLED__ */
222 std r10,PACA_EXSLB+EX_R10(r13)
223 std r11,PACA_EXSLB+EX_R11(r13)
224 std r12,PACA_EXSLB+EX_R12(r13)
225 mfspr r10,SPRN_SPRG1
226 std r10,PACA_EXSLB+EX_R13(r13)
227 mfspr r12,SPRN_SRR1 /* and SRR1 */
228#ifndef CONFIG_RELOCATABLE
229 b .slb_miss_realmode
230#else
231 /*
232 * We can't just use a direct branch to .slb_miss_realmode
233 * because the distance from here to there depends on where
234 * the kernel ends up being put.
235 */
236 mfctr r11
237 ld r10,PACAKBASE(r13)
238 LOAD_HANDLER(r10, .slb_miss_realmode)
239 mtctr r10
240 bctr
241#endif
242
243 STD_EXCEPTION_PSERIES(0x400, instruction_access)
244
245 . = 0x480
246 .globl instruction_access_slb_pSeries
247instruction_access_slb_pSeries:
248 HMT_MEDIUM
249 mtspr SPRN_SPRG1,r13
250 mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
251 std r3,PACA_EXSLB+EX_R3(r13)
252 mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
253 std r9,PACA_EXSLB+EX_R9(r13) /* save r9 - r12 */
254 mfcr r9
255#ifdef __DISABLED__
256 /* Keep that around for when we re-implement dynamic VSIDs */
257 cmpdi r3,0
258 bge slb_miss_user_pseries
259#endif /* __DISABLED__ */
260 std r10,PACA_EXSLB+EX_R10(r13)
261 std r11,PACA_EXSLB+EX_R11(r13)
262 std r12,PACA_EXSLB+EX_R12(r13)
263 mfspr r10,SPRN_SPRG1
264 std r10,PACA_EXSLB+EX_R13(r13)
265 mfspr r12,SPRN_SRR1 /* and SRR1 */
266#ifndef CONFIG_RELOCATABLE
267 b .slb_miss_realmode
268#else
269 mfctr r11
270 ld r10,PACAKBASE(r13)
271 LOAD_HANDLER(r10, .slb_miss_realmode)
272 mtctr r10
273 bctr
274#endif
275
276 MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
277 STD_EXCEPTION_PSERIES(0x600, alignment)
278 STD_EXCEPTION_PSERIES(0x700, program_check)
279 STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
280 MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
281 STD_EXCEPTION_PSERIES(0xa00, trap_0a)
282 STD_EXCEPTION_PSERIES(0xb00, trap_0b)
283
284 . = 0xc00
285 .globl system_call_pSeries
286system_call_pSeries:
287 HMT_MEDIUM
288BEGIN_FTR_SECTION
289 cmpdi r0,0x1ebe
290 beq- 1f
291END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)
292 mr r9,r13
293 mfspr r13,SPRN_SPRG3
294 mfspr r11,SPRN_SRR0
295 ld r12,PACAKBASE(r13)
296 ld r10,PACAKMSR(r13)
297 LOAD_HANDLER(r12, system_call_entry)
298 mtspr SPRN_SRR0,r12
299 mfspr r12,SPRN_SRR1
300 mtspr SPRN_SRR1,r10
301 rfid
302 b . /* prevent speculative execution */
303
304/* Fast LE/BE switch system call */
3051: mfspr r12,SPRN_SRR1
306 xori r12,r12,MSR_LE
307 mtspr SPRN_SRR1,r12
308 rfid /* return to userspace */
309 b .
310
311 STD_EXCEPTION_PSERIES(0xd00, single_step)
312 STD_EXCEPTION_PSERIES(0xe00, trap_0e)
313
314 /* We need to deal with the Altivec unavailable exception
315 * here which is at 0xf20, thus in the middle of the
316 * prolog code of the PerformanceMonitor one. A little
317 * trickery is thus necessary
318 */
319 . = 0xf00
320 b performance_monitor_pSeries
321
322 . = 0xf20
323 b altivec_unavailable_pSeries
324
325 . = 0xf40
326 b vsx_unavailable_pSeries
327
328#ifdef CONFIG_CBE_RAS
329 HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
330#endif /* CONFIG_CBE_RAS */
331 STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
332#ifdef CONFIG_CBE_RAS
333 HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
334#endif /* CONFIG_CBE_RAS */
335 STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
336#ifdef CONFIG_CBE_RAS
337 HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
338#endif /* CONFIG_CBE_RAS */
339
340 . = 0x3000
341
342/*** pSeries interrupt support ***/
343
344 /* moved from 0xf00 */
345 STD_EXCEPTION_PSERIES(., performance_monitor)
346 STD_EXCEPTION_PSERIES(., altivec_unavailable)
347 STD_EXCEPTION_PSERIES(., vsx_unavailable)
348
349/*
350 * An interrupt came in while soft-disabled; clear EE in SRR1,
351 * clear paca->hard_enabled and return.
352 */
353masked_interrupt:
354 stb r10,PACAHARDIRQEN(r13)
355 mtcrf 0x80,r9
356 ld r9,PACA_EXGEN+EX_R9(r13)
357 mfspr r10,SPRN_SRR1
358 rldicl r10,r10,48,1 /* clear MSR_EE */
359 rotldi r10,r10,16
360 mtspr SPRN_SRR1,r10
361 ld r10,PACA_EXGEN+EX_R10(r13)
362 mfspr r13,SPRN_SPRG1
363 rfid
364 b .
365
366 .align 7
367do_stab_bolted_pSeries:
368 mtcrf 0x80,r12
369 mfspr r12,SPRN_SPRG2
370 EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
371
372#ifdef CONFIG_PPC_PSERIES
373/*
374 * Vectors for the FWNMI option. Share common code.
375 */
376 .globl system_reset_fwnmi
377 .align 7
378system_reset_fwnmi:
379 HMT_MEDIUM
380 mtspr SPRN_SPRG1,r13 /* save r13 */
381 EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
382
383 .globl machine_check_fwnmi
384 .align 7
385machine_check_fwnmi:
386 HMT_MEDIUM
387 mtspr SPRN_SPRG1,r13 /* save r13 */
388 EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
389
390#endif /* CONFIG_PPC_PSERIES */
391
392#ifdef __DISABLED__
393/*
394 * This is used for when the SLB miss handler has to go virtual,
395 * which doesn't happen for now anymore but will once we re-implement
396 * dynamic VSIDs for shared page tables
397 */
398slb_miss_user_pseries:
399 std r10,PACA_EXGEN+EX_R10(r13)
400 std r11,PACA_EXGEN+EX_R11(r13)
401 std r12,PACA_EXGEN+EX_R12(r13)
402 mfspr r10,SPRG1
403 ld r11,PACA_EXSLB+EX_R9(r13)
404 ld r12,PACA_EXSLB+EX_R3(r13)
405 std r10,PACA_EXGEN+EX_R13(r13)
406 std r11,PACA_EXGEN+EX_R9(r13)
407 std r12,PACA_EXGEN+EX_R3(r13)
408 clrrdi r12,r13,32
409 mfmsr r10
410 mfspr r11,SRR0 /* save SRR0 */
411 ori r12,r12,slb_miss_user_common@l /* virt addr of handler */
412 ori r10,r10,MSR_IR|MSR_DR|MSR_RI
413 mtspr SRR0,r12
414 mfspr r12,SRR1 /* and SRR1 */
415 mtspr SRR1,r10
416 rfid
417 b . /* prevent spec. execution */
418#endif /* __DISABLED__ */
419
420 .align 7
421 .globl __end_interrupts
422__end_interrupts:
423
424/*
425 * Code from here down to __end_handlers is invoked from the
426 * exception prologs above. Because the prologs assemble the
427 * addresses of these handlers using the LOAD_HANDLER macro,
428 * which uses an addi instruction, these handlers must be in
429 * the first 32k of the kernel image.
430 */
431
432/*** Common interrupt handlers ***/
433
434 STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
435
436 /*
437 * Machine check is different because we use a different
438 * save area: PACA_EXMC instead of PACA_EXGEN.
439 */
440 .align 7
441 .globl machine_check_common
442machine_check_common:
443 EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
444 FINISH_NAP
445 DISABLE_INTS
446 bl .save_nvgprs
447 addi r3,r1,STACK_FRAME_OVERHEAD
448 bl .machine_check_exception
449 b .ret_from_except
450
451 STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
452 STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
453 STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
454 STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
455 STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
456 STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
457 STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
458#ifdef CONFIG_ALTIVEC
459 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
460#else
461 STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
462#endif
463#ifdef CONFIG_CBE_RAS
464 STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
465 STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
466 STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
467#endif /* CONFIG_CBE_RAS */
468
469 .align 7
470system_call_entry:
471 b system_call_common
472
473/*
474 * Here we have detected that the kernel stack pointer is bad.
475 * R9 contains the saved CR, r13 points to the paca,
476 * r10 contains the (bad) kernel stack pointer,
477 * r11 and r12 contain the saved SRR0 and SRR1.
478 * We switch to using an emergency stack, save the registers there,
479 * and call kernel_bad_stack(), which panics.
480 */
481bad_stack:
482 ld r1,PACAEMERGSP(r13)
483 subi r1,r1,64+INT_FRAME_SIZE
484 std r9,_CCR(r1)
485 std r10,GPR1(r1)
486 std r11,_NIP(r1)
487 std r12,_MSR(r1)
488 mfspr r11,SPRN_DAR
489 mfspr r12,SPRN_DSISR
490 std r11,_DAR(r1)
491 std r12,_DSISR(r1)
492 mflr r10
493 mfctr r11
494 mfxer r12
495 std r10,_LINK(r1)
496 std r11,_CTR(r1)
497 std r12,_XER(r1)
498 SAVE_GPR(0,r1)
499 SAVE_GPR(2,r1)
500 SAVE_4GPRS(3,r1)
501 SAVE_2GPRS(7,r1)
502 SAVE_10GPRS(12,r1)
503 SAVE_10GPRS(22,r1)
504 lhz r12,PACA_TRAP_SAVE(r13)
505 std r12,_TRAP(r1)
506 addi r11,r1,INT_FRAME_SIZE
507 std r11,0(r1)
508 li r12,0
509 std r12,0(r11)
510 ld r2,PACATOC(r13)
5111: addi r3,r1,STACK_FRAME_OVERHEAD
512 bl .kernel_bad_stack
513 b 1b
514
515/*
516 * Here r13 points to the paca, r9 contains the saved CR,
517 * SRR0 and SRR1 are saved in r11 and r12,
518 * r9 - r13 are saved in paca->exgen.
519 */
520 .align 7
521 .globl data_access_common
522data_access_common:
523 mfspr r10,SPRN_DAR
524 std r10,PACA_EXGEN+EX_DAR(r13)
525 mfspr r10,SPRN_DSISR
526 stw r10,PACA_EXGEN+EX_DSISR(r13)
527 EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
528 ld r3,PACA_EXGEN+EX_DAR(r13)
529 lwz r4,PACA_EXGEN+EX_DSISR(r13)
530 li r5,0x300
531 b .do_hash_page /* Try to handle as hpte fault */
532
533 .align 7
534 .globl instruction_access_common
535instruction_access_common:
536 EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
537 ld r3,_NIP(r1)
538 andis. r4,r12,0x5820
539 li r5,0x400
540 b .do_hash_page /* Try to handle as hpte fault */
541
542/*
543 * Here is the common SLB miss user that is used when going to virtual
544 * mode for SLB misses, that is currently not used
545 */
546#ifdef __DISABLED__
547 .align 7
548 .globl slb_miss_user_common
549slb_miss_user_common:
550 mflr r10
551 std r3,PACA_EXGEN+EX_DAR(r13)
552 stw r9,PACA_EXGEN+EX_CCR(r13)
553 std r10,PACA_EXGEN+EX_LR(r13)
554 std r11,PACA_EXGEN+EX_SRR0(r13)
555 bl .slb_allocate_user
556
557 ld r10,PACA_EXGEN+EX_LR(r13)
558 ld r3,PACA_EXGEN+EX_R3(r13)
559 lwz r9,PACA_EXGEN+EX_CCR(r13)
560 ld r11,PACA_EXGEN+EX_SRR0(r13)
561 mtlr r10
562 beq- slb_miss_fault
563
564 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
565 beq- unrecov_user_slb
566 mfmsr r10
567
568.machine push
569.machine "power4"
570 mtcrf 0x80,r9
571.machine pop
572
573 clrrdi r10,r10,2 /* clear RI before setting SRR0/1 */
574 mtmsrd r10,1
575
576 mtspr SRR0,r11
577 mtspr SRR1,r12
578
579 ld r9,PACA_EXGEN+EX_R9(r13)
580 ld r10,PACA_EXGEN+EX_R10(r13)
581 ld r11,PACA_EXGEN+EX_R11(r13)
582 ld r12,PACA_EXGEN+EX_R12(r13)
583 ld r13,PACA_EXGEN+EX_R13(r13)
584 rfid
585 b .
586
587slb_miss_fault:
588 EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
589 ld r4,PACA_EXGEN+EX_DAR(r13)
590 li r5,0
591 std r4,_DAR(r1)
592 std r5,_DSISR(r1)
593 b handle_page_fault
594
595unrecov_user_slb:
596 EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
597 DISABLE_INTS
598 bl .save_nvgprs
5991: addi r3,r1,STACK_FRAME_OVERHEAD
600 bl .unrecoverable_exception
601 b 1b
602
603#endif /* __DISABLED__ */
604
605
606/*
607 * r13 points to the PACA, r9 contains the saved CR,
608 * r12 contain the saved SRR1, SRR0 is still ready for return
609 * r3 has the faulting address
610 * r9 - r13 are saved in paca->exslb.
611 * r3 is saved in paca->slb_r3
612 * We assume we aren't going to take any exceptions during this procedure.
613 */
614_GLOBAL(slb_miss_realmode)
615 mflr r10
616#ifdef CONFIG_RELOCATABLE
617 mtctr r11
618#endif
619
620 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
621 std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
622
623 bl .slb_allocate_realmode
624
625 /* All done -- return from exception. */
626
627 ld r10,PACA_EXSLB+EX_LR(r13)
628 ld r3,PACA_EXSLB+EX_R3(r13)
629 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
630#ifdef CONFIG_PPC_ISERIES
631BEGIN_FW_FTR_SECTION
632 ld r11,PACALPPACAPTR(r13)
633 ld r11,LPPACASRR0(r11) /* get SRR0 value */
634END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
635#endif /* CONFIG_PPC_ISERIES */
636
637 mtlr r10
638
639 andi. r10,r12,MSR_RI /* check for unrecoverable exception */
640 beq- 2f
641
642.machine push
643.machine "power4"
644 mtcrf 0x80,r9
645 mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
646.machine pop
647
648#ifdef CONFIG_PPC_ISERIES
649BEGIN_FW_FTR_SECTION
650 mtspr SPRN_SRR0,r11
651 mtspr SPRN_SRR1,r12
652END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
653#endif /* CONFIG_PPC_ISERIES */
654 ld r9,PACA_EXSLB+EX_R9(r13)
655 ld r10,PACA_EXSLB+EX_R10(r13)
656 ld r11,PACA_EXSLB+EX_R11(r13)
657 ld r12,PACA_EXSLB+EX_R12(r13)
658 ld r13,PACA_EXSLB+EX_R13(r13)
659 rfid
660 b . /* prevent speculative execution */
661
6622:
663#ifdef CONFIG_PPC_ISERIES
664BEGIN_FW_FTR_SECTION
665 b unrecov_slb
666END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
667#endif /* CONFIG_PPC_ISERIES */
668 mfspr r11,SPRN_SRR0
669 ld r10,PACAKBASE(r13)
670 LOAD_HANDLER(r10,unrecov_slb)
671 mtspr SPRN_SRR0,r10
672 ld r10,PACAKMSR(r13)
673 mtspr SPRN_SRR1,r10
674 rfid
675 b .
676
677unrecov_slb:
678 EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
679 DISABLE_INTS
680 bl .save_nvgprs
6811: addi r3,r1,STACK_FRAME_OVERHEAD
682 bl .unrecoverable_exception
683 b 1b
684
685 .align 7
686 .globl hardware_interrupt_common
687 .globl hardware_interrupt_entry
688hardware_interrupt_common:
689 EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
690 FINISH_NAP
691hardware_interrupt_entry:
692 DISABLE_INTS
693BEGIN_FTR_SECTION
694 bl .ppc64_runlatch_on
695END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
696 addi r3,r1,STACK_FRAME_OVERHEAD
697 bl .do_IRQ
698 b .ret_from_except_lite
699
700#ifdef CONFIG_PPC_970_NAP
701power4_fixup_nap:
702 andc r9,r9,r10
703 std r9,TI_LOCAL_FLAGS(r11)
704 ld r10,_LINK(r1) /* make idle task do the */
705 std r10,_NIP(r1) /* equivalent of a blr */
706 blr
707#endif
708
709 .align 7
710 .globl alignment_common
711alignment_common:
712 mfspr r10,SPRN_DAR
713 std r10,PACA_EXGEN+EX_DAR(r13)
714 mfspr r10,SPRN_DSISR
715 stw r10,PACA_EXGEN+EX_DSISR(r13)
716 EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
717 ld r3,PACA_EXGEN+EX_DAR(r13)
718 lwz r4,PACA_EXGEN+EX_DSISR(r13)
719 std r3,_DAR(r1)
720 std r4,_DSISR(r1)
721 bl .save_nvgprs
722 addi r3,r1,STACK_FRAME_OVERHEAD
723 ENABLE_INTS
724 bl .alignment_exception
725 b .ret_from_except
726
727 .align 7
728 .globl program_check_common
729program_check_common:
730 EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
731 bl .save_nvgprs
732 addi r3,r1,STACK_FRAME_OVERHEAD
733 ENABLE_INTS
734 bl .program_check_exception
735 b .ret_from_except
736
737 .align 7
738 .globl fp_unavailable_common
739fp_unavailable_common:
740 EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
741 bne 1f /* if from user, just load it up */
742 bl .save_nvgprs
743 addi r3,r1,STACK_FRAME_OVERHEAD
744 ENABLE_INTS
745 bl .kernel_fp_unavailable_exception
746 BUG_OPCODE
7471: bl .load_up_fpu
748 b fast_exception_return
749
750 .align 7
751 .globl altivec_unavailable_common
752altivec_unavailable_common:
753 EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
754#ifdef CONFIG_ALTIVEC
755BEGIN_FTR_SECTION
756 beq 1f
757 bl .load_up_altivec
758 b fast_exception_return
7591:
760END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
761#endif
762 bl .save_nvgprs
763 addi r3,r1,STACK_FRAME_OVERHEAD
764 ENABLE_INTS
765 bl .altivec_unavailable_exception
766 b .ret_from_except
767
768 .align 7
769 .globl vsx_unavailable_common
770vsx_unavailable_common:
771 EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
772#ifdef CONFIG_VSX
773BEGIN_FTR_SECTION
774 bne .load_up_vsx
7751:
776END_FTR_SECTION_IFSET(CPU_FTR_VSX)
777#endif
778 bl .save_nvgprs
779 addi r3,r1,STACK_FRAME_OVERHEAD
780 ENABLE_INTS
781 bl .vsx_unavailable_exception
782 b .ret_from_except
783
784 .align 7
785 .globl __end_handlers
786__end_handlers:
787
788/*
789 * Return from an exception with minimal checks.
790 * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
791 * If interrupts have been enabled, or anything has been
792 * done that might have changed the scheduling status of
793 * any task or sent any task a signal, you should use
794 * ret_from_except or ret_from_except_lite instead of this.
795 */ 162 */
796fast_exc_return_irq: /* restores irq state too */ 163#ifdef CONFIG_PPC_BOOK3S
797 ld r3,SOFTE(r1) 164#include "exceptions-64s.S"
798 TRACE_AND_RESTORE_IRQ(r3);
799 ld r12,_MSR(r1)
800 rldicl r4,r12,49,63 /* get MSR_EE to LSB */
801 stb r4,PACAHARDIRQEN(r13) /* restore paca->hard_enabled */
802 b 1f
803
804 .globl fast_exception_return
805fast_exception_return:
806 ld r12,_MSR(r1)
8071: ld r11,_NIP(r1)
808 andi. r3,r12,MSR_RI /* check if RI is set */
809 beq- unrecov_fer
810
811#ifdef CONFIG_VIRT_CPU_ACCOUNTING
812 andi. r3,r12,MSR_PR
813 beq 2f
814 ACCOUNT_CPU_USER_EXIT(r3, r4)
8152:
816#endif 165#endif
817 166
818 ld r3,_CCR(r1)
819 ld r4,_LINK(r1)
820 ld r5,_CTR(r1)
821 ld r6,_XER(r1)
822 mtcr r3
823 mtlr r4
824 mtctr r5
825 mtxer r6
826 REST_GPR(0, r1)
827 REST_8GPRS(2, r1)
828
829 mfmsr r10
830 rldicl r10,r10,48,1 /* clear EE */
831 rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */
832 mtmsrd r10,1
833
834 mtspr SPRN_SRR1,r12
835 mtspr SPRN_SRR0,r11
836 REST_4GPRS(10, r1)
837 ld r1,GPR1(r1)
838 rfid
839 b . /* prevent speculative execution */
840
841unrecov_fer:
842 bl .save_nvgprs
8431: addi r3,r1,STACK_FRAME_OVERHEAD
844 bl .unrecoverable_exception
845 b 1b
846
847
848/*
849 * Hash table stuff
850 */
851 .align 7
852_STATIC(do_hash_page)
853 std r3,_DAR(r1)
854 std r4,_DSISR(r1)
855
856 andis. r0,r4,0xa450 /* weird error? */
857 bne- handle_page_fault /* if not, try to insert a HPTE */
858BEGIN_FTR_SECTION
859 andis. r0,r4,0x0020 /* Is it a segment table fault? */
860 bne- do_ste_alloc /* If so handle it */
861END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
862
863 /*
864 * On iSeries, we soft-disable interrupts here, then
865 * hard-enable interrupts so that the hash_page code can spin on
866 * the hash_table_lock without problems on a shared processor.
867 */
868 DISABLE_INTS
869
870 /*
871 * Currently, trace_hardirqs_off() will be called by DISABLE_INTS
872 * and will clobber volatile registers when irq tracing is enabled
873 * so we need to reload them. It may be possible to be smarter here
874 * and move the irq tracing elsewhere but let's keep it simple for
875 * now
876 */
877#ifdef CONFIG_TRACE_IRQFLAGS
878 ld r3,_DAR(r1)
879 ld r4,_DSISR(r1)
880 ld r5,_TRAP(r1)
881 ld r12,_MSR(r1)
882 clrrdi r5,r5,4
883#endif /* CONFIG_TRACE_IRQFLAGS */
884 /*
885 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
886 * accessing a userspace segment (even from the kernel). We assume
887 * kernel addresses always have the high bit set.
888 */
889 rlwinm r4,r4,32-25+9,31-9,31-9 /* DSISR_STORE -> _PAGE_RW */
890 rotldi r0,r3,15 /* Move high bit into MSR_PR posn */
891 orc r0,r12,r0 /* MSR_PR | ~high_bit */
892 rlwimi r4,r0,32-13,30,30 /* becomes _PAGE_USER access bit */
893 ori r4,r4,1 /* add _PAGE_PRESENT */
894 rlwimi r4,r5,22+2,31-2,31-2 /* Set _PAGE_EXEC if trap is 0x400 */
895
896 /*
897 * r3 contains the faulting address
898 * r4 contains the required access permissions
899 * r5 contains the trap number
900 *
901 * at return r3 = 0 for success
902 */
903 bl .hash_page /* build HPTE if possible */
904 cmpdi r3,0 /* see if hash_page succeeded */
905
906BEGIN_FW_FTR_SECTION
907 /*
908 * If we had interrupts soft-enabled at the point where the
909 * DSI/ISI occurred, and an interrupt came in during hash_page,
910 * handle it now.
911 * We jump to ret_from_except_lite rather than fast_exception_return
912 * because ret_from_except_lite will check for and handle pending
913 * interrupts if necessary.
914 */
915 beq 13f
916END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
917
918BEGIN_FW_FTR_SECTION
919 /*
920 * Here we have interrupts hard-disabled, so it is sufficient
921 * to restore paca->{soft,hard}_enable and get out.
922 */
923 beq fast_exc_return_irq /* Return from exception on success */
924END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
925
926 /* For a hash failure, we don't bother re-enabling interrupts */
927 ble- 12f
928
929 /*
930 * hash_page couldn't handle it, set soft interrupt enable back
931 * to what it was before the trap. Note that .raw_local_irq_restore
932 * handles any interrupts pending at this point.
933 */
934 ld r3,SOFTE(r1)
935 TRACE_AND_RESTORE_IRQ_PARTIAL(r3, 11f)
936 bl .raw_local_irq_restore
937 b 11f
938
939/* Here we have a page fault that hash_page can't handle. */
940handle_page_fault:
941 ENABLE_INTS
94211: ld r4,_DAR(r1)
943 ld r5,_DSISR(r1)
944 addi r3,r1,STACK_FRAME_OVERHEAD
945 bl .do_page_fault
946 cmpdi r3,0
947 beq+ 13f
948 bl .save_nvgprs
949 mr r5,r3
950 addi r3,r1,STACK_FRAME_OVERHEAD
951 lwz r4,_DAR(r1)
952 bl .bad_page_fault
953 b .ret_from_except
954
95513: b .ret_from_except_lite
956
957/* We have a page fault that hash_page could handle but HV refused
958 * the PTE insertion
959 */
96012: bl .save_nvgprs
961 mr r5,r3
962 addi r3,r1,STACK_FRAME_OVERHEAD
963 ld r4,_DAR(r1)
964 bl .low_hash_fault
965 b .ret_from_except
966
967 /* here we have a segment miss */
968do_ste_alloc:
969 bl .ste_allocate /* try to insert stab entry */
970 cmpdi r3,0
971 bne- handle_page_fault
972 b fast_exception_return
973
974/*
975 * r13 points to the PACA, r9 contains the saved CR,
976 * r11 and r12 contain the saved SRR0 and SRR1.
977 * r9 - r13 are saved in paca->exslb.
978 * We assume we aren't going to take any exceptions during this procedure.
979 * We assume (DAR >> 60) == 0xc.
980 */
981 .align 7
982_GLOBAL(do_stab_bolted)
983 stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
984 std r11,PACA_EXSLB+EX_SRR0(r13) /* save SRR0 in exc. frame */
985
986 /* Hash to the primary group */
987 ld r10,PACASTABVIRT(r13)
988 mfspr r11,SPRN_DAR
989 srdi r11,r11,28
990 rldimi r10,r11,7,52 /* r10 = first ste of the group */
991
992 /* Calculate VSID */
993 /* This is a kernel address, so protovsid = ESID */
994 ASM_VSID_SCRAMBLE(r11, r9, 256M)
995 rldic r9,r11,12,16 /* r9 = vsid << 12 */
996
997 /* Search the primary group for a free entry */
9981: ld r11,0(r10) /* Test valid bit of the current ste */
999 andi. r11,r11,0x80
1000 beq 2f
1001 addi r10,r10,16
1002 andi. r11,r10,0x70
1003 bne 1b
1004
1005 /* Stick for only searching the primary group for now. */
1006 /* At least for now, we use a very simple random castout scheme */
1007 /* Use the TB as a random number ; OR in 1 to avoid entry 0 */
1008 mftb r11
1009 rldic r11,r11,4,57 /* r11 = (r11 << 4) & 0x70 */
1010 ori r11,r11,0x10
1011
1012 /* r10 currently points to an ste one past the group of interest */
1013 /* make it point to the randomly selected entry */
1014 subi r10,r10,128
1015 or r10,r10,r11 /* r10 is the entry to invalidate */
1016
1017 isync /* mark the entry invalid */
1018 ld r11,0(r10)
1019 rldicl r11,r11,56,1 /* clear the valid bit */
1020 rotldi r11,r11,8
1021 std r11,0(r10)
1022 sync
1023
1024 clrrdi r11,r11,28 /* Get the esid part of the ste */
1025 slbie r11
1026
10272: std r9,8(r10) /* Store the vsid part of the ste */
1028 eieio
1029
1030 mfspr r11,SPRN_DAR /* Get the new esid */
1031 clrrdi r11,r11,28 /* Permits a full 32b of ESID */
1032 ori r11,r11,0x90 /* Turn on valid and kp */
1033 std r11,0(r10) /* Put new entry back into the stab */
1034
1035 sync
1036
1037 /* All done -- return from exception. */
1038 lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
1039 ld r11,PACA_EXSLB+EX_SRR0(r13) /* get saved SRR0 */
1040
1041 andi. r10,r12,MSR_RI
1042 beq- unrecov_slb
1043
1044 mtcrf 0x80,r9 /* restore CR */
1045
1046 mfmsr r10
1047 clrrdi r10,r10,2
1048 mtmsrd r10,1
1049
1050 mtspr SPRN_SRR0,r11
1051 mtspr SPRN_SRR1,r12
1052 ld r9,PACA_EXSLB+EX_R9(r13)
1053 ld r10,PACA_EXSLB+EX_R10(r13)
1054 ld r11,PACA_EXSLB+EX_R11(r13)
1055 ld r12,PACA_EXSLB+EX_R12(r13)
1056 ld r13,PACA_EXSLB+EX_R13(r13)
1057 rfid
1058 b . /* prevent speculative execution */
1059
1060/*
1061 * Space for CPU0's segment table.
1062 *
1063 * On iSeries, the hypervisor must fill in at least one entry before
1064 * we get control (with relocate on). The address is given to the hv
1065 * as a page number (see xLparMap below), so this must be at a
1066 * fixed address (the linker can't compute (u64)&initial_stab >>
1067 * PAGE_SHIFT).
1068 */
1069 . = STAB0_OFFSET /* 0x6000 */
1070 .globl initial_stab
1071initial_stab:
1072 .space 4096
1073
1074#ifdef CONFIG_PPC_PSERIES
1075/*
1076 * Data area reserved for FWNMI option.
1077 * This address (0x7000) is fixed by the RPA.
1078 */
1079 .= 0x7000
1080 .globl fwnmi_data_area
1081fwnmi_data_area:
1082#endif /* CONFIG_PPC_PSERIES */
1083
1084 /* iSeries does not use the FWNMI stuff, so it is safe to put
1085 * this here, even if we later allow kernels that will boot on
1086 * both pSeries and iSeries */
1087#ifdef CONFIG_PPC_ISERIES
1088 . = LPARMAP_PHYS
1089 .globl xLparMap
1090xLparMap:
1091 .quad HvEsidsToMap /* xNumberEsids */
1092 .quad HvRangesToMap /* xNumberRanges */
1093 .quad STAB0_PAGE /* xSegmentTableOffs */
1094 .zero 40 /* xRsvd */
1095 /* xEsids (HvEsidsToMap entries of 2 quads) */
1096 .quad PAGE_OFFSET_ESID /* xKernelEsid */
1097 .quad PAGE_OFFSET_VSID /* xKernelVsid */
1098 .quad VMALLOC_START_ESID /* xKernelEsid */
1099 .quad VMALLOC_START_VSID /* xKernelVsid */
1100 /* xRanges (HvRangesToMap entries of 3 quads) */
1101 .quad HvPagesToMap /* xPages */
1102 .quad 0 /* xOffset */
1103 .quad PAGE_OFFSET_VSID << (SID_SHIFT - HW_PAGE_SHIFT) /* xVPN */
1104
1105#endif /* CONFIG_PPC_ISERIES */
1106
1107#ifdef CONFIG_PPC_PSERIES
1108 . = 0x8000
1109#endif /* CONFIG_PPC_PSERIES */
1110 167
1111/* 168/*
1112 * On pSeries and most other platforms, secondary processors spin 169 * On pSeries and most other platforms, secondary processors spin
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-20 22:18:12 -0400