aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc')
-rw-r--r--arch/sparc/include/asm/Kbuild5
-rw-r--r--arch/sparc/include/asm/cputime.h6
-rw-r--r--arch/sparc/include/asm/emergency-restart.h6
-rw-r--r--arch/sparc/include/asm/mutex.h9
-rw-r--r--arch/sparc/include/asm/pgtable_64.h1
-rw-r--r--arch/sparc/include/asm/serial.h6
-rw-r--r--arch/sparc/include/asm/smp_32.h5
-rw-r--r--arch/sparc/include/asm/switch_to_64.h3
-rw-r--r--arch/sparc/include/asm/tlbflush_64.h37
-rw-r--r--arch/sparc/include/uapi/asm/Kbuild1
-rw-r--r--arch/sparc/include/uapi/asm/types.h17
-rw-r--r--arch/sparc/kernel/smp_64.c41
-rw-r--r--arch/sparc/lib/bitext.c6
-rw-r--r--arch/sparc/mm/iommu.c2
-rw-r--r--arch/sparc/mm/srmmu.c4
-rw-r--r--arch/sparc/mm/tlb.c38
-rw-r--r--arch/sparc/mm/tsb.c57
-rw-r--r--arch/sparc/mm/ultra.S119
18 files changed, 251 insertions, 112 deletions
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index e26d430ce2fd..ff18e3cfb6b1 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -2,11 +2,16 @@
2 2
3 3
4generic-y += clkdev.h 4generic-y += clkdev.h
5generic-y += cputime.h
5generic-y += div64.h 6generic-y += div64.h
7generic-y += emergency-restart.h
6generic-y += exec.h 8generic-y += exec.h
7generic-y += local64.h 9generic-y += local64.h
10generic-y += mutex.h
8generic-y += irq_regs.h 11generic-y += irq_regs.h
9generic-y += local.h 12generic-y += local.h
10generic-y += module.h 13generic-y += module.h
14generic-y += serial.h
11generic-y += trace_clock.h 15generic-y += trace_clock.h
16generic-y += types.h
12generic-y += word-at-a-time.h 17generic-y += word-at-a-time.h
diff --git a/arch/sparc/include/asm/cputime.h b/arch/sparc/include/asm/cputime.h
deleted file mode 100644
index 1a642b81e019..000000000000
--- a/arch/sparc/include/asm/cputime.h
+++ /dev/null
@@ -1,6 +0,0 @@
1#ifndef __SPARC_CPUTIME_H
2#define __SPARC_CPUTIME_H
3
4#include <asm-generic/cputime.h>
5
6#endif /* __SPARC_CPUTIME_H */
diff --git a/arch/sparc/include/asm/emergency-restart.h b/arch/sparc/include/asm/emergency-restart.h
deleted file mode 100644
index 108d8c48e42e..000000000000
--- a/arch/sparc/include/asm/emergency-restart.h
+++ /dev/null
@@ -1,6 +0,0 @@
1#ifndef _ASM_EMERGENCY_RESTART_H
2#define _ASM_EMERGENCY_RESTART_H
3
4#include <asm-generic/emergency-restart.h>
5
6#endif /* _ASM_EMERGENCY_RESTART_H */
diff --git a/arch/sparc/include/asm/mutex.h b/arch/sparc/include/asm/mutex.h
deleted file mode 100644
index 458c1f7fbc18..000000000000
--- a/arch/sparc/include/asm/mutex.h
+++ /dev/null
@@ -1,9 +0,0 @@
1/*
2 * Pull in the generic implementation for the mutex fastpath.
3 *
4 * TODO: implement optimized primitives instead, or leave the generic
5 * implementation in place, or pick the atomic_xchg() based generic
6 * implementation. (see asm-generic/mutex-xchg.h for details)
7 */
8
9#include <asm-generic/mutex-dec.h>
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 08fcce90316b..7619f2f792af 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -915,6 +915,7 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma,
915 return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); 915 return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot);
916} 916}
917 917
918#include <asm/tlbflush.h>
918#include <asm-generic/pgtable.h> 919#include <asm-generic/pgtable.h>
919 920
920/* We provide our own get_unmapped_area to cope with VA holes and 921/* We provide our own get_unmapped_area to cope with VA holes and
diff --git a/arch/sparc/include/asm/serial.h b/arch/sparc/include/asm/serial.h
deleted file mode 100644
index f90d61c28059..000000000000
--- a/arch/sparc/include/asm/serial.h
+++ /dev/null
@@ -1,6 +0,0 @@
1#ifndef __SPARC_SERIAL_H
2#define __SPARC_SERIAL_H
3
4#define BASE_BAUD ( 1843200 / 16 )
5
6#endif /* __SPARC_SERIAL_H */
diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h
index b73da3c5f10a..3c8917f054de 100644
--- a/arch/sparc/include/asm/smp_32.h
+++ b/arch/sparc/include/asm/smp_32.h
@@ -36,7 +36,6 @@ typedef void (*smpfunc_t)(unsigned long, unsigned long, unsigned long,
36 unsigned long, unsigned long); 36 unsigned long, unsigned long);
37 37
38void cpu_panic(void); 38void cpu_panic(void);
39extern void smp4m_irq_rotate(int cpu);
40 39
41/* 40/*
42 * General functions that each host system must provide. 41 * General functions that each host system must provide.
@@ -46,7 +45,6 @@ void sun4m_init_smp(void);
46void sun4d_init_smp(void); 45void sun4d_init_smp(void);
47 46
48void smp_callin(void); 47void smp_callin(void);
49void smp_boot_cpus(void);
50void smp_store_cpu_info(int); 48void smp_store_cpu_info(int);
51 49
52void smp_resched_interrupt(void); 50void smp_resched_interrupt(void);
@@ -107,9 +105,6 @@ extern int hard_smp_processor_id(void);
107 105
108#define raw_smp_processor_id() (current_thread_info()->cpu) 106#define raw_smp_processor_id() (current_thread_info()->cpu)
109 107
110#define prof_multiplier(__cpu) cpu_data(__cpu).multiplier
111#define prof_counter(__cpu) cpu_data(__cpu).counter
112
113void smp_setup_cpu_possible_map(void); 108void smp_setup_cpu_possible_map(void);
114 109
115#endif /* !(__ASSEMBLY__) */ 110#endif /* !(__ASSEMBLY__) */
diff --git a/arch/sparc/include/asm/switch_to_64.h b/arch/sparc/include/asm/switch_to_64.h
index cad36f56fa03..c7de3323819c 100644
--- a/arch/sparc/include/asm/switch_to_64.h
+++ b/arch/sparc/include/asm/switch_to_64.h
@@ -18,8 +18,7 @@ do { \
18 * and 2 stores in this critical code path. -DaveM 18 * and 2 stores in this critical code path. -DaveM
19 */ 19 */
20#define switch_to(prev, next, last) \ 20#define switch_to(prev, next, last) \
21do { flush_tlb_pending(); \ 21do { save_and_clear_fpu(); \
22 save_and_clear_fpu(); \
23 /* If you are tempted to conditionalize the following */ \ 22 /* If you are tempted to conditionalize the following */ \
24 /* so that ASI is only written if it changes, think again. */ \ 23 /* so that ASI is only written if it changes, think again. */ \
25 __asm__ __volatile__("wr %%g0, %0, %%asi" \ 24 __asm__ __volatile__("wr %%g0, %0, %%asi" \
diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h
index 2ef463494153..f0d6a9700f4c 100644
--- a/arch/sparc/include/asm/tlbflush_64.h
+++ b/arch/sparc/include/asm/tlbflush_64.h
@@ -11,24 +11,40 @@
11struct tlb_batch { 11struct tlb_batch {
12 struct mm_struct *mm; 12 struct mm_struct *mm;
13 unsigned long tlb_nr; 13 unsigned long tlb_nr;
14 unsigned long active;
14 unsigned long vaddrs[TLB_BATCH_NR]; 15 unsigned long vaddrs[TLB_BATCH_NR];
15}; 16};
16 17
17extern void flush_tsb_kernel_range(unsigned long start, unsigned long end); 18extern void flush_tsb_kernel_range(unsigned long start, unsigned long end);
18extern void flush_tsb_user(struct tlb_batch *tb); 19extern void flush_tsb_user(struct tlb_batch *tb);
20extern void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr);
19 21
20/* TLB flush operations. */ 22/* TLB flush operations. */
21 23
22extern void flush_tlb_pending(void); 24static inline void flush_tlb_mm(struct mm_struct *mm)
25{
26}
27
28static inline void flush_tlb_page(struct vm_area_struct *vma,
29 unsigned long vmaddr)
30{
31}
32
33static inline void flush_tlb_range(struct vm_area_struct *vma,
34 unsigned long start, unsigned long end)
35{
36}
37
38#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
23 39
24#define flush_tlb_range(vma,start,end) \ 40extern void flush_tlb_pending(void);
25 do { (void)(start); flush_tlb_pending(); } while (0) 41extern void arch_enter_lazy_mmu_mode(void);
26#define flush_tlb_page(vma,addr) flush_tlb_pending() 42extern void arch_leave_lazy_mmu_mode(void);
27#define flush_tlb_mm(mm) flush_tlb_pending() 43#define arch_flush_lazy_mmu_mode() do {} while (0)
28 44
29/* Local cpu only. */ 45/* Local cpu only. */
30extern void __flush_tlb_all(void); 46extern void __flush_tlb_all(void);
31 47extern void __flush_tlb_page(unsigned long context, unsigned long vaddr);
32extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end); 48extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end);
33 49
34#ifndef CONFIG_SMP 50#ifndef CONFIG_SMP
@@ -38,15 +54,24 @@ do { flush_tsb_kernel_range(start,end); \
38 __flush_tlb_kernel_range(start,end); \ 54 __flush_tlb_kernel_range(start,end); \
39} while (0) 55} while (0)
40 56
57static inline void global_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr)
58{
59 __flush_tlb_page(CTX_HWBITS(mm->context), vaddr);
60}
61
41#else /* CONFIG_SMP */ 62#else /* CONFIG_SMP */
42 63
43extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); 64extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end);
65extern void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr);
44 66
45#define flush_tlb_kernel_range(start, end) \ 67#define flush_tlb_kernel_range(start, end) \
46do { flush_tsb_kernel_range(start,end); \ 68do { flush_tsb_kernel_range(start,end); \
47 smp_flush_tlb_kernel_range(start, end); \ 69 smp_flush_tlb_kernel_range(start, end); \
48} while (0) 70} while (0)
49 71
72#define global_flush_tlb_page(mm, vaddr) \
73 smp_flush_tlb_page(mm, vaddr)
74
50#endif /* ! CONFIG_SMP */ 75#endif /* ! CONFIG_SMP */
51 76
52#endif /* _SPARC64_TLBFLUSH_H */ 77#endif /* _SPARC64_TLBFLUSH_H */
diff --git a/arch/sparc/include/uapi/asm/Kbuild b/arch/sparc/include/uapi/asm/Kbuild
index ce175aff71b7..b5843ee09fb5 100644
--- a/arch/sparc/include/uapi/asm/Kbuild
+++ b/arch/sparc/include/uapi/asm/Kbuild
@@ -44,7 +44,6 @@ header-y += swab.h
44header-y += termbits.h 44header-y += termbits.h
45header-y += termios.h 45header-y += termios.h
46header-y += traps.h 46header-y += traps.h
47header-y += types.h
48header-y += uctx.h 47header-y += uctx.h
49header-y += unistd.h 48header-y += unistd.h
50header-y += utrap.h 49header-y += utrap.h
diff --git a/arch/sparc/include/uapi/asm/types.h b/arch/sparc/include/uapi/asm/types.h
deleted file mode 100644
index 383d156cde9c..000000000000
--- a/arch/sparc/include/uapi/asm/types.h
+++ /dev/null
@@ -1,17 +0,0 @@
1#ifndef _SPARC_TYPES_H
2#define _SPARC_TYPES_H
3/*
4 * This file is never included by application software unless
5 * explicitly requested (e.g., via linux/types.h) in which case the
6 * application is Linux specific so (user-) name space pollution is
7 * not a major issue. However, for interoperability, libraries still
8 * need to be careful to avoid a name clashes.
9 */
10
11#if defined(__sparc__)
12
13#include <asm-generic/int-ll64.h>
14
15#endif /* defined(__sparc__) */
16
17#endif /* defined(_SPARC_TYPES_H) */
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 537eb66abd06..ca64d2a86ec0 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -849,7 +849,7 @@ void smp_tsb_sync(struct mm_struct *mm)
849} 849}
850 850
851extern unsigned long xcall_flush_tlb_mm; 851extern unsigned long xcall_flush_tlb_mm;
852extern unsigned long xcall_flush_tlb_pending; 852extern unsigned long xcall_flush_tlb_page;
853extern unsigned long xcall_flush_tlb_kernel_range; 853extern unsigned long xcall_flush_tlb_kernel_range;
854extern unsigned long xcall_fetch_glob_regs; 854extern unsigned long xcall_fetch_glob_regs;
855extern unsigned long xcall_fetch_glob_pmu; 855extern unsigned long xcall_fetch_glob_pmu;
@@ -1074,23 +1074,56 @@ local_flush_and_out:
1074 put_cpu(); 1074 put_cpu();
1075} 1075}
1076 1076
1077struct tlb_pending_info {
1078 unsigned long ctx;
1079 unsigned long nr;
1080 unsigned long *vaddrs;
1081};
1082
1083static void tlb_pending_func(void *info)
1084{
1085 struct tlb_pending_info *t = info;
1086
1087 __flush_tlb_pending(t->ctx, t->nr, t->vaddrs);
1088}
1089
1077void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs) 1090void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs)
1078{ 1091{
1079 u32 ctx = CTX_HWBITS(mm->context); 1092 u32 ctx = CTX_HWBITS(mm->context);
1093 struct tlb_pending_info info;
1080 int cpu = get_cpu(); 1094 int cpu = get_cpu();
1081 1095
1096 info.ctx = ctx;
1097 info.nr = nr;
1098 info.vaddrs = vaddrs;
1099
1082 if (mm == current->mm && atomic_read(&mm->mm_users) == 1) 1100 if (mm == current->mm && atomic_read(&mm->mm_users) == 1)
1083 cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); 1101 cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
1084 else 1102 else
1085 smp_cross_call_masked(&xcall_flush_tlb_pending, 1103 smp_call_function_many(mm_cpumask(mm), tlb_pending_func,
1086 ctx, nr, (unsigned long) vaddrs, 1104 &info, 1);
1087 mm_cpumask(mm));
1088 1105
1089 __flush_tlb_pending(ctx, nr, vaddrs); 1106 __flush_tlb_pending(ctx, nr, vaddrs);
1090 1107
1091 put_cpu(); 1108 put_cpu();
1092} 1109}
1093 1110
1111void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr)
1112{
1113 unsigned long context = CTX_HWBITS(mm->context);
1114 int cpu = get_cpu();
1115
1116 if (mm == current->mm && atomic_read(&mm->mm_users) == 1)
1117 cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
1118 else
1119 smp_cross_call_masked(&xcall_flush_tlb_page,
1120 context, vaddr, 0,
1121 mm_cpumask(mm));
1122 __flush_tlb_page(context, vaddr);
1123
1124 put_cpu();
1125}
1126
1094void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end) 1127void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end)
1095{ 1128{
1096 start &= PAGE_MASK; 1129 start &= PAGE_MASK;
diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c
index 48d00e72ce15..8ec4e9c0251a 100644
--- a/arch/sparc/lib/bitext.c
+++ b/arch/sparc/lib/bitext.c
@@ -119,11 +119,7 @@ void bit_map_clear(struct bit_map *t, int offset, int len)
119 119
120void bit_map_init(struct bit_map *t, unsigned long *map, int size) 120void bit_map_init(struct bit_map *t, unsigned long *map, int size)
121{ 121{
122 122 bitmap_zero(map, size);
123 if ((size & 07) != 0)
124 BUG();
125 memset(map, 0, size>>3);
126
127 memset(t, 0, sizeof *t); 123 memset(t, 0, sizeof *t);
128 spin_lock_init(&t->lock); 124 spin_lock_init(&t->lock);
129 t->map = map; 125 t->map = map;
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index 0f4f7191fbba..28f96f27c768 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -34,7 +34,7 @@
34#define IOMMU_RNGE IOMMU_RNGE_256MB 34#define IOMMU_RNGE IOMMU_RNGE_256MB
35#define IOMMU_START 0xF0000000 35#define IOMMU_START 0xF0000000
36#define IOMMU_WINSIZE (256*1024*1024U) 36#define IOMMU_WINSIZE (256*1024*1024U)
37#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */ 37#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
38#define IOMMU_ORDER 6 /* 4096 * (1<<6) */ 38#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
39 39
40/* srmmu.c */ 40/* srmmu.c */
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index c38bb72e3e80..036c2797dece 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -280,7 +280,9 @@ static void __init srmmu_nocache_init(void)
280 SRMMU_NOCACHE_ALIGN_MAX, 0UL); 280 SRMMU_NOCACHE_ALIGN_MAX, 0UL);
281 memset(srmmu_nocache_pool, 0, srmmu_nocache_size); 281 memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
282 282
283 srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL); 283 srmmu_nocache_bitmap =
284 __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
285 SMP_CACHE_BYTES, 0UL);
284 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); 286 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
285 287
286 srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); 288 srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index ba6ae7ffdc2c..272aa4f7657e 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -24,11 +24,17 @@ static DEFINE_PER_CPU(struct tlb_batch, tlb_batch);
24void flush_tlb_pending(void) 24void flush_tlb_pending(void)
25{ 25{
26 struct tlb_batch *tb = &get_cpu_var(tlb_batch); 26 struct tlb_batch *tb = &get_cpu_var(tlb_batch);
27 struct mm_struct *mm = tb->mm;
27 28
28 if (tb->tlb_nr) { 29 if (!tb->tlb_nr)
29 flush_tsb_user(tb); 30 goto out;
30 31
31 if (CTX_VALID(tb->mm->context)) { 32 flush_tsb_user(tb);
33
34 if (CTX_VALID(mm->context)) {
35 if (tb->tlb_nr == 1) {
36 global_flush_tlb_page(mm, tb->vaddrs[0]);
37 } else {
32#ifdef CONFIG_SMP 38#ifdef CONFIG_SMP
33 smp_flush_tlb_pending(tb->mm, tb->tlb_nr, 39 smp_flush_tlb_pending(tb->mm, tb->tlb_nr,
34 &tb->vaddrs[0]); 40 &tb->vaddrs[0]);
@@ -37,12 +43,30 @@ void flush_tlb_pending(void)
37 tb->tlb_nr, &tb->vaddrs[0]); 43 tb->tlb_nr, &tb->vaddrs[0]);
38#endif 44#endif
39 } 45 }
40 tb->tlb_nr = 0;
41 } 46 }
42 47
48 tb->tlb_nr = 0;
49
50out:
43 put_cpu_var(tlb_batch); 51 put_cpu_var(tlb_batch);
44} 52}
45 53
54void arch_enter_lazy_mmu_mode(void)
55{
56 struct tlb_batch *tb = &__get_cpu_var(tlb_batch);
57
58 tb->active = 1;
59}
60
61void arch_leave_lazy_mmu_mode(void)
62{
63 struct tlb_batch *tb = &__get_cpu_var(tlb_batch);
64
65 if (tb->tlb_nr)
66 flush_tlb_pending();
67 tb->active = 0;
68}
69
46static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, 70static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
47 bool exec) 71 bool exec)
48{ 72{
@@ -60,6 +84,12 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
60 nr = 0; 84 nr = 0;
61 } 85 }
62 86
87 if (!tb->active) {
88 global_flush_tlb_page(mm, vaddr);
89 flush_tsb_user_page(mm, vaddr);
90 return;
91 }
92
63 if (nr == 0) 93 if (nr == 0)
64 tb->mm = mm; 94 tb->mm = mm;
65 95
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 428982b9becf..2cc3bce5ee91 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -7,11 +7,10 @@
7#include <linux/preempt.h> 7#include <linux/preempt.h>
8#include <linux/slab.h> 8#include <linux/slab.h>
9#include <asm/page.h> 9#include <asm/page.h>
10#include <asm/tlbflush.h>
11#include <asm/tlb.h>
12#include <asm/mmu_context.h>
13#include <asm/pgtable.h> 10#include <asm/pgtable.h>
11#include <asm/mmu_context.h>
14#include <asm/tsb.h> 12#include <asm/tsb.h>
13#include <asm/tlb.h>
15#include <asm/oplib.h> 14#include <asm/oplib.h>
16 15
17extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; 16extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
@@ -46,23 +45,27 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
46 } 45 }
47} 46}
48 47
49static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, 48static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v,
50 unsigned long tsb, unsigned long nentries) 49 unsigned long hash_shift,
50 unsigned long nentries)
51{ 51{
52 unsigned long i; 52 unsigned long tag, ent, hash;
53 53
54 for (i = 0; i < tb->tlb_nr; i++) { 54 v &= ~0x1UL;
55 unsigned long v = tb->vaddrs[i]; 55 hash = tsb_hash(v, hash_shift, nentries);
56 unsigned long tag, ent, hash; 56 ent = tsb + (hash * sizeof(struct tsb));
57 tag = (v >> 22UL);
57 58
58 v &= ~0x1UL; 59 tsb_flush(ent, tag);
60}
59 61
60 hash = tsb_hash(v, hash_shift, nentries); 62static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift,
61 ent = tsb + (hash * sizeof(struct tsb)); 63 unsigned long tsb, unsigned long nentries)
62 tag = (v >> 22UL); 64{
65 unsigned long i;
63 66
64 tsb_flush(ent, tag); 67 for (i = 0; i < tb->tlb_nr; i++)
65 } 68 __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries);
66} 69}
67 70
68void flush_tsb_user(struct tlb_batch *tb) 71void flush_tsb_user(struct tlb_batch *tb)
@@ -90,6 +93,30 @@ void flush_tsb_user(struct tlb_batch *tb)
90 spin_unlock_irqrestore(&mm->context.lock, flags); 93 spin_unlock_irqrestore(&mm->context.lock, flags);
91} 94}
92 95
96void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
97{
98 unsigned long nentries, base, flags;
99
100 spin_lock_irqsave(&mm->context.lock, flags);
101
102 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
103 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
104 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
105 base = __pa(base);
106 __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
107
108#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
109 if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
110 base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
111 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
112 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
113 base = __pa(base);
114 __flush_tsb_one_entry(base, vaddr, HPAGE_SHIFT, nentries);
115 }
116#endif
117 spin_unlock_irqrestore(&mm->context.lock, flags);
118}
119
93#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K 120#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K
94#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K 121#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K
95 122
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index f8e13d421fcb..432aa0cb1b38 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -53,6 +53,33 @@ __flush_tlb_mm: /* 18 insns */
53 nop 53 nop
54 54
55 .align 32 55 .align 32
56 .globl __flush_tlb_page
57__flush_tlb_page: /* 22 insns */
58 /* %o0 = context, %o1 = vaddr */
59 rdpr %pstate, %g7
60 andn %g7, PSTATE_IE, %g2
61 wrpr %g2, %pstate
62 mov SECONDARY_CONTEXT, %o4
63 ldxa [%o4] ASI_DMMU, %g2
64 stxa %o0, [%o4] ASI_DMMU
65 andcc %o1, 1, %g0
66 andn %o1, 1, %o3
67 be,pn %icc, 1f
68 or %o3, 0x10, %o3
69 stxa %g0, [%o3] ASI_IMMU_DEMAP
701: stxa %g0, [%o3] ASI_DMMU_DEMAP
71 membar #Sync
72 stxa %g2, [%o4] ASI_DMMU
73 sethi %hi(KERNBASE), %o4
74 flush %o4
75 retl
76 wrpr %g7, 0x0, %pstate
77 nop
78 nop
79 nop
80 nop
81
82 .align 32
56 .globl __flush_tlb_pending 83 .globl __flush_tlb_pending
57__flush_tlb_pending: /* 26 insns */ 84__flush_tlb_pending: /* 26 insns */
58 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 85 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
@@ -203,6 +230,31 @@ __cheetah_flush_tlb_mm: /* 19 insns */
203 retl 230 retl
204 wrpr %g7, 0x0, %pstate 231 wrpr %g7, 0x0, %pstate
205 232
233__cheetah_flush_tlb_page: /* 22 insns */
234 /* %o0 = context, %o1 = vaddr */
235 rdpr %pstate, %g7
236 andn %g7, PSTATE_IE, %g2
237 wrpr %g2, 0x0, %pstate
238 wrpr %g0, 1, %tl
239 mov PRIMARY_CONTEXT, %o4
240 ldxa [%o4] ASI_DMMU, %g2
241 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
242 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
243 or %o0, %o3, %o0 /* Preserve nucleus page size fields */
244 stxa %o0, [%o4] ASI_DMMU
245 andcc %o1, 1, %g0
246 be,pn %icc, 1f
247 andn %o1, 1, %o3
248 stxa %g0, [%o3] ASI_IMMU_DEMAP
2491: stxa %g0, [%o3] ASI_DMMU_DEMAP
250 membar #Sync
251 stxa %g2, [%o4] ASI_DMMU
252 sethi %hi(KERNBASE), %o4
253 flush %o4
254 wrpr %g0, 0, %tl
255 retl
256 wrpr %g7, 0x0, %pstate
257
206__cheetah_flush_tlb_pending: /* 27 insns */ 258__cheetah_flush_tlb_pending: /* 27 insns */
207 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 259 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
208 rdpr %pstate, %g7 260 rdpr %pstate, %g7
@@ -269,6 +321,20 @@ __hypervisor_flush_tlb_mm: /* 10 insns */
269 retl 321 retl
270 nop 322 nop
271 323
324__hypervisor_flush_tlb_page: /* 11 insns */
325 /* %o0 = context, %o1 = vaddr */
326 mov %o0, %g2
327 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
328 mov %g2, %o1 /* ARG1: mmu context */
329 mov HV_MMU_ALL, %o2 /* ARG2: flags */
330 srlx %o0, PAGE_SHIFT, %o0
331 sllx %o0, PAGE_SHIFT, %o0
332 ta HV_MMU_UNMAP_ADDR_TRAP
333 brnz,pn %o0, __hypervisor_tlb_tl0_error
334 mov HV_MMU_UNMAP_ADDR_TRAP, %o1
335 retl
336 nop
337
272__hypervisor_flush_tlb_pending: /* 16 insns */ 338__hypervisor_flush_tlb_pending: /* 16 insns */
273 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 339 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
274 sllx %o1, 3, %g1 340 sllx %o1, 3, %g1
@@ -339,6 +405,13 @@ cheetah_patch_cachetlbops:
339 call tlb_patch_one 405 call tlb_patch_one
340 mov 19, %o2 406 mov 19, %o2
341 407
408 sethi %hi(__flush_tlb_page), %o0
409 or %o0, %lo(__flush_tlb_page), %o0
410 sethi %hi(__cheetah_flush_tlb_page), %o1
411 or %o1, %lo(__cheetah_flush_tlb_page), %o1
412 call tlb_patch_one
413 mov 22, %o2
414
342 sethi %hi(__flush_tlb_pending), %o0 415 sethi %hi(__flush_tlb_pending), %o0
343 or %o0, %lo(__flush_tlb_pending), %o0 416 or %o0, %lo(__flush_tlb_pending), %o0
344 sethi %hi(__cheetah_flush_tlb_pending), %o1 417 sethi %hi(__cheetah_flush_tlb_pending), %o1
@@ -397,10 +470,9 @@ xcall_flush_tlb_mm: /* 21 insns */
397 nop 470 nop
398 nop 471 nop
399 472
400 .globl xcall_flush_tlb_pending 473 .globl xcall_flush_tlb_page
401xcall_flush_tlb_pending: /* 21 insns */ 474xcall_flush_tlb_page: /* 17 insns */
402 /* %g5=context, %g1=nr, %g7=vaddrs[] */ 475 /* %g5=context, %g1=vaddr */
403 sllx %g1, 3, %g1
404 mov PRIMARY_CONTEXT, %g4 476 mov PRIMARY_CONTEXT, %g4
405 ldxa [%g4] ASI_DMMU, %g2 477 ldxa [%g4] ASI_DMMU, %g2
406 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 478 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
@@ -408,20 +480,16 @@ xcall_flush_tlb_pending: /* 21 insns */
408 or %g5, %g4, %g5 480 or %g5, %g4, %g5
409 mov PRIMARY_CONTEXT, %g4 481 mov PRIMARY_CONTEXT, %g4
410 stxa %g5, [%g4] ASI_DMMU 482 stxa %g5, [%g4] ASI_DMMU
4111: sub %g1, (1 << 3), %g1 483 andcc %g1, 0x1, %g0
412 ldx [%g7 + %g1], %g5
413 andcc %g5, 0x1, %g0
414 be,pn %icc, 2f 484 be,pn %icc, 2f
415 485 andn %g1, 0x1, %g5
416 andn %g5, 0x1, %g5
417 stxa %g0, [%g5] ASI_IMMU_DEMAP 486 stxa %g0, [%g5] ASI_IMMU_DEMAP
4182: stxa %g0, [%g5] ASI_DMMU_DEMAP 4872: stxa %g0, [%g5] ASI_DMMU_DEMAP
419 membar #Sync 488 membar #Sync
420 brnz,pt %g1, 1b
421 nop
422 stxa %g2, [%g4] ASI_DMMU 489 stxa %g2, [%g4] ASI_DMMU
423 retry 490 retry
424 nop 491 nop
492 nop
425 493
426 .globl xcall_flush_tlb_kernel_range 494 .globl xcall_flush_tlb_kernel_range
427xcall_flush_tlb_kernel_range: /* 25 insns */ 495xcall_flush_tlb_kernel_range: /* 25 insns */
@@ -656,15 +724,13 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
656 membar #Sync 724 membar #Sync
657 retry 725 retry
658 726
659 .globl __hypervisor_xcall_flush_tlb_pending 727 .globl __hypervisor_xcall_flush_tlb_page
660__hypervisor_xcall_flush_tlb_pending: /* 21 insns */ 728__hypervisor_xcall_flush_tlb_page: /* 17 insns */
661 /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */ 729 /* %g5=ctx, %g1=vaddr */
662 sllx %g1, 3, %g1
663 mov %o0, %g2 730 mov %o0, %g2
664 mov %o1, %g3 731 mov %o1, %g3
665 mov %o2, %g4 732 mov %o2, %g4
6661: sub %g1, (1 << 3), %g1 733 mov %g1, %o0 /* ARG0: virtual address */
667 ldx [%g7 + %g1], %o0 /* ARG0: virtual address */
668 mov %g5, %o1 /* ARG1: mmu context */ 734 mov %g5, %o1 /* ARG1: mmu context */
669 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 735 mov HV_MMU_ALL, %o2 /* ARG2: flags */
670 srlx %o0, PAGE_SHIFT, %o0 736 srlx %o0, PAGE_SHIFT, %o0
@@ -673,8 +739,6 @@ __hypervisor_xcall_flush_tlb_pending: /* 21 insns */
673 mov HV_MMU_UNMAP_ADDR_TRAP, %g6 739 mov HV_MMU_UNMAP_ADDR_TRAP, %g6
674 brnz,a,pn %o0, __hypervisor_tlb_xcall_error 740 brnz,a,pn %o0, __hypervisor_tlb_xcall_error
675 mov %o0, %g5 741 mov %o0, %g5
676 brnz,pt %g1, 1b
677 nop
678 mov %g2, %o0 742 mov %g2, %o0
679 mov %g3, %o1 743 mov %g3, %o1
680 mov %g4, %o2 744 mov %g4, %o2
@@ -757,6 +821,13 @@ hypervisor_patch_cachetlbops:
757 call tlb_patch_one 821 call tlb_patch_one
758 mov 10, %o2 822 mov 10, %o2
759 823
824 sethi %hi(__flush_tlb_page), %o0
825 or %o0, %lo(__flush_tlb_page), %o0
826 sethi %hi(__hypervisor_flush_tlb_page), %o1
827 or %o1, %lo(__hypervisor_flush_tlb_page), %o1
828 call tlb_patch_one
829 mov 11, %o2
830
760 sethi %hi(__flush_tlb_pending), %o0 831 sethi %hi(__flush_tlb_pending), %o0
761 or %o0, %lo(__flush_tlb_pending), %o0 832 or %o0, %lo(__flush_tlb_pending), %o0
762 sethi %hi(__hypervisor_flush_tlb_pending), %o1 833 sethi %hi(__hypervisor_flush_tlb_pending), %o1
@@ -788,12 +859,12 @@ hypervisor_patch_cachetlbops:
788 call tlb_patch_one 859 call tlb_patch_one
789 mov 21, %o2 860 mov 21, %o2
790 861
791 sethi %hi(xcall_flush_tlb_pending), %o0 862 sethi %hi(xcall_flush_tlb_page), %o0
792 or %o0, %lo(xcall_flush_tlb_pending), %o0 863 or %o0, %lo(xcall_flush_tlb_page), %o0
793 sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1 864 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
794 or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1 865 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
795 call tlb_patch_one 866 call tlb_patch_one
796 mov 21, %o2 867 mov 17, %o2
797 868
798 sethi %hi(xcall_flush_tlb_kernel_range), %o0 869 sethi %hi(xcall_flush_tlb_kernel_range), %o0
799 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 870 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-27 03:21:49 -0500