aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc64')
-rw-r--r--arch/sparc64/kernel/entry.S122
-rw-r--r--arch/sparc64/kernel/etrap.S18
-rw-r--r--arch/sparc64/kernel/head.S20
-rw-r--r--arch/sparc64/kernel/irq.c26
-rw-r--r--arch/sparc64/kernel/rtrap.S10
-rw-r--r--arch/sparc64/kernel/setup.c8
-rw-r--r--arch/sparc64/kernel/smp.c55
-rw-r--r--arch/sparc64/kernel/trampoline.S9
-rw-r--r--arch/sparc64/kernel/traps.c19
-rw-r--r--arch/sparc64/kernel/tsb.S26
-rw-r--r--arch/sparc64/kernel/ttable.S2
-rw-r--r--arch/sparc64/kernel/winfixup.S24
-rw-r--r--arch/sparc64/mm/ultra.S10
13 files changed, 192 insertions, 157 deletions
diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index a73553ae7e53..906b64ffdb1b 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -50,7 +50,8 @@ do_fpdis:
50 add %g0, %g0, %g0 50 add %g0, %g0, %g0
51 ba,a,pt %xcc, rtrap_clr_l6 51 ba,a,pt %xcc, rtrap_clr_l6
52 52
531: ldub [%g6 + TI_FPSAVED], %g5 531: TRAP_LOAD_THREAD_REG
54 ldub [%g6 + TI_FPSAVED], %g5
54 wr %g0, FPRS_FEF, %fprs 55 wr %g0, FPRS_FEF, %fprs
55 andcc %g5, FPRS_FEF, %g0 56 andcc %g5, FPRS_FEF, %g0
56 be,a,pt %icc, 1f 57 be,a,pt %icc, 1f
@@ -189,6 +190,7 @@ fp_other_bounce:
189 .globl do_fpother_check_fitos 190 .globl do_fpother_check_fitos
190 .align 32 191 .align 32
191do_fpother_check_fitos: 192do_fpother_check_fitos:
193 TRAP_LOAD_THREAD_REG
192 sethi %hi(fp_other_bounce - 4), %g7 194 sethi %hi(fp_other_bounce - 4), %g7
193 or %g7, %lo(fp_other_bounce - 4), %g7 195 or %g7, %lo(fp_other_bounce - 4), %g7
194 196
@@ -353,8 +355,6 @@ do_fptrap_after_fsr:
353 * 355 *
354 * With this method we can do most of the cross-call tlb/cache 356 * With this method we can do most of the cross-call tlb/cache
355 * flushing very quickly. 357 * flushing very quickly.
356 *
357 * Current CPU's IRQ worklist table is locked into %g6, don't touch.
358 */ 358 */
359 .text 359 .text
360 .align 32 360 .align 32
@@ -378,6 +378,8 @@ do_ivec:
378 sllx %g2, %g4, %g2 378 sllx %g2, %g4, %g2
379 sllx %g4, 2, %g4 379 sllx %g4, 2, %g4
380 380
381 TRAP_LOAD_IRQ_WORK
382
381 lduw [%g6 + %g4], %g5 /* g5 = irq_work(cpu, pil) */ 383 lduw [%g6 + %g4], %g5 /* g5 = irq_work(cpu, pil) */
382 stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */ 384 stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */
383 stw %g3, [%g6 + %g4] /* irq_work(cpu, pil) = bucket */ 385 stw %g3, [%g6 + %g4] /* irq_work(cpu, pil) = bucket */
@@ -488,9 +490,24 @@ setcc:
488 retl 490 retl
489 stx %o1, [%o0 + PT_V9_TSTATE] 491 stx %o1, [%o0 + PT_V9_TSTATE]
490 492
491 .globl utrap, utrap_ill 493 .globl utrap_trap
492utrap: brz,pn %g1, etrap 494utrap_trap: /* %g3=handler,%g4=level */
495 TRAP_LOAD_THREAD_REG
496 ldx [%g6 + TI_UTRAPS], %g1
497 brnz,pt %g1, invoke_utrap
493 nop 498 nop
499
500 ba,pt %xcc, etrap
501 rd %pc, %g7
502 mov %l4, %o1
503 call bad_trap
504 add %sp, PTREGS_OFF, %o0
505 ba,pt %xcc, rtrap
506 clr %l6
507
508invoke_utrap:
509 sllx %g3, 3, %g3
510 ldx [%g1 + %g3], %g1
494 save %sp, -128, %sp 511 save %sp, -128, %sp
495 rdpr %tstate, %l6 512 rdpr %tstate, %l6
496 rdpr %cwp, %l7 513 rdpr %cwp, %l7
@@ -500,17 +517,6 @@ utrap: brz,pn %g1, etrap
500 rdpr %tnpc, %l7 517 rdpr %tnpc, %l7
501 wrpr %g1, 0, %tnpc 518 wrpr %g1, 0, %tnpc
502 done 519 done
503utrap_ill:
504 call bad_trap
505 add %sp, PTREGS_OFF, %o0
506 ba,pt %xcc, rtrap
507 clr %l6
508
509 /* XXX Here is stuff we still need to write... -DaveM XXX */
510 .globl netbsd_syscall
511netbsd_syscall:
512 retl
513 nop
514 520
515 /* We need to carefully read the error status, ACK 521 /* We need to carefully read the error status, ACK
516 * the errors, prevent recursive traps, and pass the 522 * the errors, prevent recursive traps, and pass the
@@ -1001,7 +1007,7 @@ dcpe_icpe_tl1_common:
1001 * %g3: scratch 1007 * %g3: scratch
1002 * %g4: AFSR 1008 * %g4: AFSR
1003 * %g5: AFAR 1009 * %g5: AFAR
1004 * %g6: current thread ptr 1010 * %g6: unused, will have current thread ptr after etrap
1005 * %g7: scratch 1011 * %g7: scratch
1006 */ 1012 */
1007__cheetah_log_error: 1013__cheetah_log_error:
@@ -1690,3 +1696,85 @@ __flushw_user:
1690 restore %g0, %g0, %g0 1696 restore %g0, %g0, %g0
16912: retl 16972: retl
1692 nop 1698 nop
1699
1700 /* Read cpu ID from hardware, return in %g6.
1701 * (callers_pc - 4) is in %g1. Patched at boot time.
1702 *
1703 * Default is spitfire implementation.
1704 *
1705 * The instruction sequence needs to be 5 instructions
1706 * in order to fit the longest implementation, which is
1707 * currently starfire.
1708 */
1709 .align 32
1710 .globl __get_cpu_id
1711__get_cpu_id:
1712 ldxa [%g0] ASI_UPA_CONFIG, %g6
1713 srlx %g6, 17, %g6
1714 jmpl %g1 + 0x4, %g0
1715 and %g6, 0x1f, %g6
1716 nop
1717
1718__get_cpu_id_cheetah_safari:
1719 ldxa [%g0] ASI_SAFARI_CONFIG, %g6
1720 srlx %g6, 17, %g6
1721 jmpl %g1 + 0x4, %g0
1722 and %g6, 0x3ff, %g6
1723 nop
1724
1725__get_cpu_id_cheetah_jbus:
1726 ldxa [%g0] ASI_JBUS_CONFIG, %g6
1727 srlx %g6, 17, %g6
1728 jmpl %g1 + 0x4, %g0
1729 and %g6, 0x1f, %g6
1730 nop
1731
1732__get_cpu_id_starfire:
1733 sethi %hi(0x1fff40000d0 >> 9), %g6
1734 sllx %g6, 9, %g6
1735 or %g6, 0xd0, %g6
1736 jmpl %g1 + 0x4, %g0
1737 lduwa [%g6] ASI_PHYS_BYPASS_EC_E, %g6
1738
1739 .globl per_cpu_patch
1740per_cpu_patch:
1741 sethi %hi(this_is_starfire), %o0
1742 lduw [%o0 + %lo(this_is_starfire)], %o1
1743 sethi %hi(__get_cpu_id_starfire), %o0
1744 brnz,pn %o1, 10f
1745 or %o0, %lo(__get_cpu_id_starfire), %o0
1746 sethi %hi(tlb_type), %o0
1747 lduw [%o0 + %lo(tlb_type)], %o1
1748 brz,pt %o1, 11f
1749 nop
1750 rdpr %ver, %o0
1751 srlx %o0, 32, %o0
1752 sethi %hi(0x003e0016), %o1
1753 or %o1, %lo(0x003e0016), %o1
1754 cmp %o0, %o1
1755 sethi %hi(__get_cpu_id_cheetah_jbus), %o0
1756 be,pn %icc, 10f
1757 or %o0, %lo(__get_cpu_id_cheetah_jbus), %o0
1758 sethi %hi(__get_cpu_id_cheetah_safari), %o0
1759 or %o0, %lo(__get_cpu_id_cheetah_safari), %o0
176010:
1761 sethi %hi(__get_cpu_id), %o1
1762 or %o1, %lo(__get_cpu_id), %o1
1763 lduw [%o0 + 0x00], %o2
1764 stw %o2, [%o1 + 0x00]
1765 flush %o1 + 0x00
1766 lduw [%o0 + 0x04], %o2
1767 stw %o2, [%o1 + 0x04]
1768 flush %o1 + 0x04
1769 lduw [%o0 + 0x08], %o2
1770 stw %o2, [%o1 + 0x08]
1771 flush %o1 + 0x08
1772 lduw [%o0 + 0x0c], %o2
1773 stw %o2, [%o1 + 0x0c]
1774 flush %o1 + 0x0c
1775 lduw [%o0 + 0x10], %o2
1776 stw %o2, [%o1 + 0x10]
1777 flush %o1 + 0x10
177811:
1779 retl
1780 nop
diff --git a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S
index 567dbb765c34..8b3b6d720ed5 100644
--- a/arch/sparc64/kernel/etrap.S
+++ b/arch/sparc64/kernel/etrap.S
@@ -31,6 +31,7 @@
31 .globl etrap, etrap_irq, etraptl1 31 .globl etrap, etrap_irq, etraptl1
32etrap: rdpr %pil, %g2 32etrap: rdpr %pil, %g2
33etrap_irq: 33etrap_irq:
34 TRAP_LOAD_THREAD_REG
34 rdpr %tstate, %g1 35 rdpr %tstate, %g1
35 sllx %g2, 20, %g3 36 sllx %g2, 20, %g3
36 andcc %g1, TSTATE_PRIV, %g0 37 andcc %g1, TSTATE_PRIV, %g0
@@ -98,11 +99,7 @@ etrap_irq:
98 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] 99 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7]
99 wrpr %g0, ETRAP_PSTATE2, %pstate 100 wrpr %g0, ETRAP_PSTATE2, %pstate
100 mov %l6, %g6 101 mov %l6, %g6
101#ifdef CONFIG_SMP 102 LOAD_PER_CPU_BASE(%g4, %g3)
102#error IMMU TSB usage must be fixed
103 mov TSB_REG, %g3
104 ldxa [%g3] ASI_IMMU, %g5
105#endif
106 jmpl %l2 + 0x4, %g0 103 jmpl %l2 + 0x4, %g0
107 ldx [%g6 + TI_TASK], %g4 104 ldx [%g6 + TI_TASK], %g4
108 105
@@ -126,6 +123,7 @@ etraptl1: /* Save tstate/tpc/tnpc of TL 1-->4 and the tl register itself.
126 * 0x58 TL4's TT 123 * 0x58 TL4's TT
127 * 0x60 TL 124 * 0x60 TL
128 */ 125 */
126 TRAP_LOAD_THREAD_REG
129 sub %sp, ((4 * 8) * 4) + 8, %g2 127 sub %sp, ((4 * 8) * 4) + 8, %g2
130 rdpr %tl, %g1 128 rdpr %tl, %g1
131 129
@@ -179,7 +177,9 @@ etraptl1: /* Save tstate/tpc/tnpc of TL 1-->4 and the tl register itself.
179 177
180 .align 64 178 .align 64
181 .globl scetrap 179 .globl scetrap
182scetrap: rdpr %pil, %g2 180scetrap:
181 TRAP_LOAD_THREAD_REG
182 rdpr %pil, %g2
183 rdpr %tstate, %g1 183 rdpr %tstate, %g1
184 sllx %g2, 20, %g3 184 sllx %g2, 20, %g3
185 andcc %g1, TSTATE_PRIV, %g0 185 andcc %g1, TSTATE_PRIV, %g0
@@ -248,11 +248,7 @@ scetrap: rdpr %pil, %g2
248 stx %i6, [%sp + PTREGS_OFF + PT_V9_I6] 248 stx %i6, [%sp + PTREGS_OFF + PT_V9_I6]
249 mov %l6, %g6 249 mov %l6, %g6
250 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] 250 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7]
251#ifdef CONFIG_SMP 251 LOAD_PER_CPU_BASE(%g4, %g3)
252#error IMMU TSB usage must be fixed
253 mov TSB_REG, %g3
254 ldxa [%g3] ASI_IMMU, %g5
255#endif
256 ldx [%g6 + TI_TASK], %g4 252 ldx [%g6 + TI_TASK], %g4
257 done 253 done
258 254
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index d00e20693be1..82ce5bced9c7 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -26,6 +26,7 @@
26#include <asm/head.h> 26#include <asm/head.h>
27#include <asm/ttable.h> 27#include <asm/ttable.h>
28#include <asm/mmu.h> 28#include <asm/mmu.h>
29#include <asm/cpudata.h>
29 30
30/* This section from from _start to sparc64_boot_end should fit into 31/* This section from from _start to sparc64_boot_end should fit into
31 * 0x0000000000404000 to 0x0000000000408000. 32 * 0x0000000000404000 to 0x0000000000408000.
@@ -421,24 +422,6 @@ setup_trap_table:
421 stxa %g2, [%g1] ASI_DMMU 422 stxa %g2, [%g1] ASI_DMMU
422 membar #Sync 423 membar #Sync
423 424
424 /* The Linux trap handlers expect various trap global registers
425 * to be setup with some fixed values. So here we set these
426 * up very carefully. These globals are:
427 *
428 * Alternate Globals (PSTATE_AG):
429 *
430 * %g6 --> current_thread_info()
431 *
432 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()):
433 *
434 * %g6 --> __irq_work[smp_processor_id()]
435 */
436
437 rdpr %pstate, %o1
438 mov %g6, %o2
439 wrpr %o1, PSTATE_AG, %pstate
440 mov %o2, %g6
441
442 /* Kill PROM timer */ 425 /* Kill PROM timer */
443 sethi %hi(0x80000000), %o2 426 sethi %hi(0x80000000), %o2
444 sllx %o2, 32, %o2 427 sllx %o2, 32, %o2
@@ -457,7 +440,6 @@ setup_trap_table:
457 440
4582: 4412:
459 wrpr %g0, %g0, %wstate 442 wrpr %g0, %g0, %wstate
460 wrpr %o1, 0x0, %pstate
461 443
462 call init_irqwork_curcpu 444 call init_irqwork_curcpu
463 nop 445 nop
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index f7490ef629b9..3e48af2769d4 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -848,33 +848,9 @@ static void kill_prom_timer(void)
848 848
849void init_irqwork_curcpu(void) 849void init_irqwork_curcpu(void)
850{ 850{
851 register struct irq_work_struct *workp asm("o2");
852 register unsigned long tmp asm("o3");
853 int cpu = hard_smp_processor_id(); 851 int cpu = hard_smp_processor_id();
854 852
855 memset(__irq_work + cpu, 0, sizeof(*workp)); 853 memset(__irq_work + cpu, 0, sizeof(struct irq_work_struct));
856
857 /* Make sure we are called with PSTATE_IE disabled. */
858 __asm__ __volatile__("rdpr %%pstate, %0\n\t"
859 : "=r" (tmp));
860 if (tmp & PSTATE_IE) {
861 prom_printf("BUG: init_irqwork_curcpu() called with "
862 "PSTATE_IE enabled, bailing.\n");
863 __asm__ __volatile__("mov %%i7, %0\n\t"
864 : "=r" (tmp));
865 prom_printf("BUG: Called from %lx\n", tmp);
866 prom_halt();
867 }
868
869 /* Set interrupt globals. */
870 workp = &__irq_work[cpu];
871 __asm__ __volatile__(
872 "rdpr %%pstate, %0\n\t"
873 "wrpr %0, %1, %%pstate\n\t"
874 "mov %2, %%g6\n\t"
875 "wrpr %0, 0x0, %%pstate\n\t"
876 : "=&r" (tmp)
877 : "i" (PSTATE_IG), "r" (workp));
878} 854}
879 855
880/* Only invoked on boot processor. */ 856/* Only invoked on boot processor. */
diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S
index 213eb4a9d8a4..5a62ec5d531c 100644
--- a/arch/sparc64/kernel/rtrap.S
+++ b/arch/sparc64/kernel/rtrap.S
@@ -223,12 +223,10 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1
223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3
224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4
225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5
226#ifdef CONFIG_SMP 226 brz,pt %l3, 1f
227#error IMMU TSB usage must be fixed 227 nop
228 mov TSB_REG, %g6 228 /* Must do this before thread reg is clobbered below. */
229 brnz,a,pn %l3, 1f 229 LOAD_PER_CPU_BASE(%g6, %g7)
230 ldxa [%g6] ASI_IMMU, %g5
231#endif
2321: 2301:
233 ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 231 ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6
234 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 232 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7
diff --git a/arch/sparc64/kernel/setup.c b/arch/sparc64/kernel/setup.c
index 158bd31e15b7..59a70301a6cf 100644
--- a/arch/sparc64/kernel/setup.c
+++ b/arch/sparc64/kernel/setup.c
@@ -507,6 +507,11 @@ void __init setup_arch(char **cmdline_p)
507 /* Work out if we are starfire early on */ 507 /* Work out if we are starfire early on */
508 check_if_starfire(); 508 check_if_starfire();
509 509
510 /* Now we know enough to patch the __get_cpu_id()
511 * trampoline used by trap code.
512 */
513 per_cpu_patch();
514
510 boot_flags_init(*cmdline_p); 515 boot_flags_init(*cmdline_p);
511 516
512 idprom_init(); 517 idprom_init();
@@ -545,6 +550,9 @@ void __init setup_arch(char **cmdline_p)
545 smp_setup_cpu_possible_map(); 550 smp_setup_cpu_possible_map();
546 551
547 paging_init(); 552 paging_init();
553
554 /* Get boot processor trap_block[] setup. */
555 init_cur_cpu_trap();
548} 556}
549 557
550static int __init set_preferred_console(void) 558static int __init set_preferred_console(void)
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index d2d3369e7b5d..8c245859d212 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -38,6 +38,7 @@
38#include <asm/timer.h> 38#include <asm/timer.h>
39#include <asm/starfire.h> 39#include <asm/starfire.h>
40#include <asm/tlb.h> 40#include <asm/tlb.h>
41#include <asm/sections.h>
41 42
42extern void calibrate_delay(void); 43extern void calibrate_delay(void);
43 44
@@ -87,10 +88,6 @@ void __init smp_store_cpu_info(int id)
87 cpu_data(id).clock_tick = prom_getintdefault(cpu_node, 88 cpu_data(id).clock_tick = prom_getintdefault(cpu_node,
88 "clock-frequency", 0); 89 "clock-frequency", 0);
89 90
90 cpu_data(id).pgcache_size = 0;
91 cpu_data(id).pte_cache[0] = NULL;
92 cpu_data(id).pte_cache[1] = NULL;
93 cpu_data(id).pgd_cache = NULL;
94 cpu_data(id).idle_volume = 1; 91 cpu_data(id).idle_volume = 1;
95 92
96 cpu_data(id).dcache_size = prom_getintdefault(cpu_node, "dcache-size", 93 cpu_data(id).dcache_size = prom_getintdefault(cpu_node, "dcache-size",
@@ -121,26 +118,15 @@ static volatile unsigned long callin_flag = 0;
121 118
122extern void inherit_locked_prom_mappings(int save_p); 119extern void inherit_locked_prom_mappings(int save_p);
123 120
124static inline void cpu_setup_percpu_base(unsigned long cpu_id)
125{
126#error IMMU TSB usage must be fixed
127 __asm__ __volatile__("mov %0, %%g5\n\t"
128 "stxa %0, [%1] %2\n\t"
129 "membar #Sync"
130 : /* no outputs */
131 : "r" (__per_cpu_offset(cpu_id)),
132 "r" (TSB_REG), "i" (ASI_IMMU));
133}
134
135void __init smp_callin(void) 121void __init smp_callin(void)
136{ 122{
137 int cpuid = hard_smp_processor_id(); 123 int cpuid = hard_smp_processor_id();
138 124
139 inherit_locked_prom_mappings(0); 125 inherit_locked_prom_mappings(0);
140 126
141 __flush_tlb_all(); 127 __local_per_cpu_offset = __per_cpu_offset(cpuid);
142 128
143 cpu_setup_percpu_base(cpuid); 129 __flush_tlb_all();
144 130
145 smp_setup_percpu_timer(); 131 smp_setup_percpu_timer();
146 132
@@ -1107,12 +1093,15 @@ void __init smp_setup_cpu_possible_map(void)
1107 1093
1108void __devinit smp_prepare_boot_cpu(void) 1094void __devinit smp_prepare_boot_cpu(void)
1109{ 1095{
1110 if (hard_smp_processor_id() >= NR_CPUS) { 1096 int cpu = hard_smp_processor_id();
1097
1098 if (cpu >= NR_CPUS) {
1111 prom_printf("Serious problem, boot cpu id >= NR_CPUS\n"); 1099 prom_printf("Serious problem, boot cpu id >= NR_CPUS\n");
1112 prom_halt(); 1100 prom_halt();
1113 } 1101 }
1114 1102
1115 current_thread_info()->cpu = hard_smp_processor_id(); 1103 current_thread_info()->cpu = cpu;
1104 __local_per_cpu_offset = __per_cpu_offset(cpu);
1116 1105
1117 cpu_set(smp_processor_id(), cpu_online_map); 1106 cpu_set(smp_processor_id(), cpu_online_map);
1118 cpu_set(smp_processor_id(), phys_cpu_present_map); 1107 cpu_set(smp_processor_id(), phys_cpu_present_map);
@@ -1173,12 +1162,9 @@ void __init setup_per_cpu_areas(void)
1173{ 1162{
1174 unsigned long goal, size, i; 1163 unsigned long goal, size, i;
1175 char *ptr; 1164 char *ptr;
1176 /* Created by linker magic */
1177 extern char __per_cpu_start[], __per_cpu_end[];
1178 1165
1179 /* Copy section for each CPU (we discard the original) */ 1166 /* Copy section for each CPU (we discard the original) */
1180 goal = ALIGN(__per_cpu_end - __per_cpu_start, PAGE_SIZE); 1167 goal = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
1181
1182#ifdef CONFIG_MODULES 1168#ifdef CONFIG_MODULES
1183 if (goal < PERCPU_ENOUGH_ROOM) 1169 if (goal < PERCPU_ENOUGH_ROOM)
1184 goal = PERCPU_ENOUGH_ROOM; 1170 goal = PERCPU_ENOUGH_ROOM;
@@ -1187,31 +1173,10 @@ void __init setup_per_cpu_areas(void)
1187 for (size = 1UL; size < goal; size <<= 1UL) 1173 for (size = 1UL; size < goal; size <<= 1UL)
1188 __per_cpu_shift++; 1174 __per_cpu_shift++;
1189 1175
1190 /* Make sure the resulting __per_cpu_base value 1176 ptr = alloc_bootmem(size * NR_CPUS);
1191 * will fit in the 43-bit sign extended IMMU
1192 * TSB register.
1193 */
1194 ptr = __alloc_bootmem(size * NR_CPUS, PAGE_SIZE,
1195 (unsigned long) __per_cpu_start);
1196 1177
1197 __per_cpu_base = ptr - __per_cpu_start; 1178 __per_cpu_base = ptr - __per_cpu_start;
1198 1179
1199 if ((__per_cpu_shift < PAGE_SHIFT) ||
1200 (__per_cpu_base & ~PAGE_MASK) ||
1201 (__per_cpu_base != (((long) __per_cpu_base << 20) >> 20))) {
1202 prom_printf("PER_CPU: Invalid layout, "
1203 "ptr[%p] shift[%lx] base[%lx]\n",
1204 ptr, __per_cpu_shift, __per_cpu_base);
1205 prom_halt();
1206 }
1207
1208 for (i = 0; i < NR_CPUS; i++, ptr += size) 1180 for (i = 0; i < NR_CPUS; i++, ptr += size)
1209 memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); 1181 memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
1210
1211 /* Finally, load in the boot cpu's base value.
1212 * We abuse the IMMU TSB register for trap handler
1213 * entry and exit loading of %g5. That is why it
1214 * has to be page aligned.
1215 */
1216 cpu_setup_percpu_base(hard_smp_processor_id());
1217} 1182}
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 782d8c4973e4..18c333f841e3 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -287,21 +287,18 @@ do_unlock:
287 wrpr %g0, 0, %wstate 287 wrpr %g0, 0, %wstate
288 wrpr %g0, 0, %tl 288 wrpr %g0, 0, %tl
289 289
290 /* Setup the trap globals, then we can resurface. */ 290 /* Load TBA, then we can resurface. */
291 rdpr %pstate, %o1
292 mov %g6, %o2
293 wrpr %o1, PSTATE_AG, %pstate
294 sethi %hi(sparc64_ttable_tl0), %g5 291 sethi %hi(sparc64_ttable_tl0), %g5
295 wrpr %g5, %tba 292 wrpr %g5, %tba
296 mov %o2, %g6
297 293
298 wrpr %o1, 0x0, %pstate
299 ldx [%g6 + TI_TASK], %g4 294 ldx [%g6 + TI_TASK], %g4
300 295
301 wrpr %g0, 0, %wstate 296 wrpr %g0, 0, %wstate
302 297
303 call init_irqwork_curcpu 298 call init_irqwork_curcpu
304 nop 299 nop
300 call init_cur_cpu_trap
301 nop
305 302
306 /* Start using proper page size encodings in ctx register. */ 303 /* Start using proper page size encodings in ctx register. */
307 sethi %hi(sparc64_kern_pri_context), %g3 304 sethi %hi(sparc64_kern_pri_context), %g3
diff --git a/arch/sparc64/kernel/traps.c b/arch/sparc64/kernel/traps.c
index 8d44ae5a15e3..f47f4874253c 100644
--- a/arch/sparc64/kernel/traps.c
+++ b/arch/sparc64/kernel/traps.c
@@ -2130,7 +2130,22 @@ void do_getpsr(struct pt_regs *regs)
2130 } 2130 }
2131} 2131}
2132 2132
2133struct trap_per_cpu trap_block[NR_CPUS];
2134
2135/* This can get invoked before sched_init() so play it super safe
2136 * and use hard_smp_processor_id().
2137 */
2138void init_cur_cpu_trap(void)
2139{
2140 int cpu = hard_smp_processor_id();
2141 struct trap_per_cpu *p = &trap_block[cpu];
2142
2143 p->thread = current_thread_info();
2144 p->pgd_paddr = 0;
2145}
2146
2133extern void thread_info_offsets_are_bolixed_dave(void); 2147extern void thread_info_offsets_are_bolixed_dave(void);
2148extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2134 2149
2135/* Only invoked on boot processor. */ 2150/* Only invoked on boot processor. */
2136void __init trap_init(void) 2151void __init trap_init(void)
@@ -2165,6 +2180,10 @@ void __init trap_init(void)
2165 (TI_FPREGS & (64 - 1))) 2180 (TI_FPREGS & (64 - 1)))
2166 thread_info_offsets_are_bolixed_dave(); 2181 thread_info_offsets_are_bolixed_dave();
2167 2182
2183 if (TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu, thread) ||
2184 TRAP_PER_CPU_PGD_PADDR != offsetof(struct trap_per_cpu, pgd_paddr))
2185 trap_per_cpu_offsets_are_bolixed_dave();
2186
2168 /* Attach to the address space of init_task. On SMP we 2187 /* Attach to the address space of init_task. On SMP we
2169 * do this in smp.c:smp_callin for other cpus. 2188 * do this in smp.c:smp_callin for other cpus.
2170 */ 2189 */
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
index 44b9e6fed09f..50752c518773 100644
--- a/arch/sparc64/kernel/tsb.S
+++ b/arch/sparc64/kernel/tsb.S
@@ -36,6 +36,15 @@ tsb_miss_itlb:
36 nop 36 nop
37 37
38tsb_miss_page_table_walk: 38tsb_miss_page_table_walk:
39 /* This clobbers %g1 and %g6, preserve them... */
40 mov %g1, %g5
41 mov %g6, %g2
42
43 TRAP_LOAD_PGD_PHYS
44
45 mov %g2, %g6
46 mov %g5, %g1
47
39 USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) 48 USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
40 49
41tsb_reload: 50tsb_reload:
@@ -112,15 +121,20 @@ winfix_trampoline:
112 * %o0: page table physical address 121 * %o0: page table physical address
113 * %o1: TSB address 122 * %o1: TSB address
114 */ 123 */
124 .align 32
115 .globl tsb_context_switch 125 .globl tsb_context_switch
116tsb_context_switch: 126tsb_context_switch:
117 wrpr %g0, PSTATE_MG | PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV, %pstate 127 rdpr %pstate, %o5
128 wrpr %o5, PSTATE_IE, %pstate
118 129
119 /* Set page table base alternate global. */ 130 ldub [%g6 + TI_CPU], %o3
120 mov %o0, %g7 131 sethi %hi(trap_block), %o4
132 sllx %o3, TRAP_BLOCK_SZ_SHIFT, %o3
133 or %o4, %lo(trap_block), %o4
134 add %o4, %o3, %o4
135 stx %o0, [%o4 + TRAP_PER_CPU_PGD_PADDR]
121 136
122 /* XXX can this happen? */ 137 brgez %o1, 9f
123 brz,pn %o1, 9f
124 nop 138 nop
125 139
126 /* Lock TSB into D-TLB. */ 140 /* Lock TSB into D-TLB. */
@@ -163,7 +177,7 @@ tsb_context_switch:
163 membar #Sync 177 membar #Sync
164 178
1659: 1799:
166 wrpr %g0, PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV | PSTATE_IE, %pstate 180 wrpr %o5, %pstate
167 181
168 retl 182 retl
169 mov %o2, %o0 183 mov %o2, %o0
diff --git a/arch/sparc64/kernel/ttable.S b/arch/sparc64/kernel/ttable.S
index 56f060c8fbf0..2fb7a33993c0 100644
--- a/arch/sparc64/kernel/ttable.S
+++ b/arch/sparc64/kernel/ttable.S
@@ -128,7 +128,7 @@ tl0_flushw: FLUSH_WINDOW_TRAP
128tl0_resv104: BTRAP(0x104) BTRAP(0x105) BTRAP(0x106) BTRAP(0x107) 128tl0_resv104: BTRAP(0x104) BTRAP(0x105) BTRAP(0x106) BTRAP(0x107)
129 .globl tl0_solaris 129 .globl tl0_solaris
130tl0_solaris: SOLARIS_SYSCALL_TRAP 130tl0_solaris: SOLARIS_SYSCALL_TRAP
131tl0_netbsd: NETBSD_SYSCALL_TRAP 131tl0_resv109: BTRAP(0x109)
132tl0_resv10a: BTRAP(0x10a) BTRAP(0x10b) BTRAP(0x10c) BTRAP(0x10d) BTRAP(0x10e) 132tl0_resv10a: BTRAP(0x10a) BTRAP(0x10b) BTRAP(0x10c) BTRAP(0x10d) BTRAP(0x10e)
133tl0_resv10f: BTRAP(0x10f) 133tl0_resv10f: BTRAP(0x10f)
134tl0_linux32: LINUX_32BIT_SYSCALL_TRAP 134tl0_linux32: LINUX_32BIT_SYSCALL_TRAP
diff --git a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S
index f5d93aa99cbb..de588036df43 100644
--- a/arch/sparc64/kernel/winfixup.S
+++ b/arch/sparc64/kernel/winfixup.S
@@ -39,6 +39,7 @@ set_pcontext:
39 */ 39 */
40 .globl fill_fixup, spill_fixup 40 .globl fill_fixup, spill_fixup
41fill_fixup: 41fill_fixup:
42 TRAP_LOAD_THREAD_REG
42 rdpr %tstate, %g1 43 rdpr %tstate, %g1
43 andcc %g1, TSTATE_PRIV, %g0 44 andcc %g1, TSTATE_PRIV, %g0
44 or %g4, FAULT_CODE_WINFIXUP, %g4 45 or %g4, FAULT_CODE_WINFIXUP, %g4
@@ -84,11 +85,7 @@ fill_fixup:
84 wrpr %l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate 85 wrpr %l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate
85 mov %o7, %g6 86 mov %o7, %g6
86 ldx [%g6 + TI_TASK], %g4 87 ldx [%g6 + TI_TASK], %g4
87#ifdef CONFIG_SMP 88 LOAD_PER_CPU_BASE(%g1, %g2)
88#error IMMU TSB usage must be fixed
89 mov TSB_REG, %g1
90 ldxa [%g1] ASI_IMMU, %g5
91#endif
92 89
93 /* This is the same as below, except we handle this a bit special 90 /* This is the same as below, except we handle this a bit special
94 * since we must preserve %l5 and %l6, see comment above. 91 * since we must preserve %l5 and %l6, see comment above.
@@ -107,6 +104,7 @@ fill_fixup:
107 * do not touch %g7 or %g2 so we handle the two cases fine. 104 * do not touch %g7 or %g2 so we handle the two cases fine.
108 */ 105 */
109spill_fixup: 106spill_fixup:
107 TRAP_LOAD_THREAD_REG
110 ldx [%g6 + TI_FLAGS], %g1 108 ldx [%g6 + TI_FLAGS], %g1
111 andcc %g1, _TIF_32BIT, %g0 109 andcc %g1, _TIF_32BIT, %g0
112 ldub [%g6 + TI_WSAVED], %g1 110 ldub [%g6 + TI_WSAVED], %g1
@@ -182,6 +180,7 @@ winfix_mna:
182 wrpr %g3, %tnpc 180 wrpr %g3, %tnpc
183 done 181 done
184fill_fixup_mna: 182fill_fixup_mna:
183 TRAP_LOAD_THREAD_REG
185 rdpr %tstate, %g1 184 rdpr %tstate, %g1
186 andcc %g1, TSTATE_PRIV, %g0 185 andcc %g1, TSTATE_PRIV, %g0
187 be,pt %xcc, window_mna_from_user_common 186 be,pt %xcc, window_mna_from_user_common
@@ -209,17 +208,14 @@ fill_fixup_mna:
209 wrpr %l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate 208 wrpr %l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate
210 mov %o7, %g6 ! Get current back. 209 mov %o7, %g6 ! Get current back.
211 ldx [%g6 + TI_TASK], %g4 ! Finish it. 210 ldx [%g6 + TI_TASK], %g4 ! Finish it.
212#ifdef CONFIG_SMP 211 LOAD_PER_CPU_BASE(%g1, %g2)
213#error IMMU TSB usage must be fixed
214 mov TSB_REG, %g1
215 ldxa [%g1] ASI_IMMU, %g5
216#endif
217 call mem_address_unaligned 212 call mem_address_unaligned
218 add %sp, PTREGS_OFF, %o0 213 add %sp, PTREGS_OFF, %o0
219 214
220 b,pt %xcc, rtrap 215 b,pt %xcc, rtrap
221 nop ! yes, the nop is correct 216 nop ! yes, the nop is correct
222spill_fixup_mna: 217spill_fixup_mna:
218 TRAP_LOAD_THREAD_REG
223 ldx [%g6 + TI_FLAGS], %g1 219 ldx [%g6 + TI_FLAGS], %g1
224 andcc %g1, _TIF_32BIT, %g0 220 andcc %g1, _TIF_32BIT, %g0
225 ldub [%g6 + TI_WSAVED], %g1 221 ldub [%g6 + TI_WSAVED], %g1
@@ -287,6 +283,7 @@ winfix_dax:
287 wrpr %g3, %tnpc 283 wrpr %g3, %tnpc
288 done 284 done
289fill_fixup_dax: 285fill_fixup_dax:
286 TRAP_LOAD_THREAD_REG
290 rdpr %tstate, %g1 287 rdpr %tstate, %g1
291 andcc %g1, TSTATE_PRIV, %g0 288 andcc %g1, TSTATE_PRIV, %g0
292 be,pt %xcc, window_dax_from_user_common 289 be,pt %xcc, window_dax_from_user_common
@@ -314,17 +311,14 @@ fill_fixup_dax:
314 wrpr %l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate 311 wrpr %l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate
315 mov %o7, %g6 ! Get current back. 312 mov %o7, %g6 ! Get current back.
316 ldx [%g6 + TI_TASK], %g4 ! Finish it. 313 ldx [%g6 + TI_TASK], %g4 ! Finish it.
317#ifdef CONFIG_SMP 314 LOAD_PER_CPU_BASE(%g1, %g2)
318#error IMMU TSB usage must be fixed
319 mov TSB_REG, %g1
320 ldxa [%g1] ASI_IMMU, %g5
321#endif
322 call spitfire_data_access_exception 315 call spitfire_data_access_exception
323 add %sp, PTREGS_OFF, %o0 316 add %sp, PTREGS_OFF, %o0
324 317
325 b,pt %xcc, rtrap 318 b,pt %xcc, rtrap
326 nop ! yes, the nop is correct 319 nop ! yes, the nop is correct
327spill_fixup_dax: 320spill_fixup_dax:
321 TRAP_LOAD_THREAD_REG
328 ldx [%g6 + TI_FLAGS], %g1 322 ldx [%g6 + TI_FLAGS], %g1
329 andcc %g1, _TIF_32BIT, %g0 323 andcc %g1, _TIF_32BIT, %g0
330 ldub [%g6 + TI_WSAVED], %g1 324 ldub [%g6 + TI_WSAVED], %g1
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index 22791f29552e..a87394824ec2 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -295,12 +295,10 @@ cheetah_patch_cachetlbops:
295 * %g1 address arg 1 (tlb page and range flushes) 295 * %g1 address arg 1 (tlb page and range flushes)
296 * %g7 address arg 2 (tlb range flush only) 296 * %g7 address arg 2 (tlb range flush only)
297 * 297 *
298 * %g6 ivector table, don't touch 298 * %g6 scratch 1
299 * %g2 scratch 1 299 * %g2 scratch 2
300 * %g3 scratch 2 300 * %g3 scratch 3
301 * %g4 scratch 3 301 * %g4 scratch 4
302 *
303 * TODO: Make xcall TLB range flushes use the tricks above... -DaveM
304 */ 302 */
305 .align 32 303 .align 32
306 .globl xcall_flush_tlb_mm 304 .globl xcall_flush_tlb_mm
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-12 12:35:54 -0500