aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arc/include/asm/mmu.h2
-rw-r--r--arch/arc/include/asm/mmu_context.h97
-rw-r--r--arch/arc/mm/tlb.c22
-rw-r--r--arch/arc/mm/tlbex.S5
4 files changed, 40 insertions, 86 deletions
diff --git a/arch/arc/include/asm/mmu.h b/arch/arc/include/asm/mmu.h
index 1639f25e47b1..c82db8bd7270 100644
--- a/arch/arc/include/asm/mmu.h
+++ b/arch/arc/include/asm/mmu.h
@@ -48,7 +48,7 @@
48#ifndef __ASSEMBLY__ 48#ifndef __ASSEMBLY__
49 49
50typedef struct { 50typedef struct {
51 unsigned long asid; /* Pvt Addr-Space ID for mm */ 51 unsigned long asid; /* 8 bit MMU PID + Generation cycle */
52} mm_context_t; 52} mm_context_t;
53 53
54#ifdef CONFIG_ARC_DBG_TLB_PARANOIA 54#ifdef CONFIG_ARC_DBG_TLB_PARANOIA
diff --git a/arch/arc/include/asm/mmu_context.h b/arch/arc/include/asm/mmu_context.h
index 9b09d18f01b3..43a1b51bb8cc 100644
--- a/arch/arc/include/asm/mmu_context.h
+++ b/arch/arc/include/asm/mmu_context.h
@@ -34,39 +34,22 @@
34 * When it reaches max 255, the allocation cycle starts afresh by flushing 34 * When it reaches max 255, the allocation cycle starts afresh by flushing
35 * the entire TLB and wrapping ASID back to zero. 35 * the entire TLB and wrapping ASID back to zero.
36 * 36 *
37 * For book-keeping, Linux uses a couple of data-structures: 37 * A new allocation cycle, post rollover, could potentially reassign an ASID
38 * -mm_struct has an @asid field to keep a note of task's ASID (needed at the 38 * to a different task. Thus the rule is to refresh the ASID in a new cycle.
39 * time of say switch_mm( ) 39 * The 32 bit @asid_cache (and mm->asid) have 8 bits MMU PID and rest 24 bits
40 * -An array of mm structs @asid_mm_map[] for asid->mm the reverse mapping, 40 * serve as cycle/generation indicator and natural 32 bit unsigned math
41 * given an ASID, finding the mm struct associated. 41 * automagically increments the generation when lower 8 bits rollover.
42 *
43 * The round-robin allocation algorithm allows for ASID stealing.
44 * If asid tracker is at "x-1", a new req will allocate "x", even if "x" was
45 * already assigned to another (switched-out) task. Obviously the prev owner
46 * is marked with an invalid ASID to make it request for a new ASID when it
47 * gets scheduled next time. However its TLB entries (with ASID "x") could
48 * exist, which must be cleared before the same ASID is used by the new owner.
49 * Flushing them would be plausible but costly solution. Instead we force a
50 * allocation policy quirk, which ensures that a stolen ASID won't have any
51 * TLB entries associates, alleviating the need to flush.
52 * The quirk essentially is not allowing ASID allocated in prev cycle
53 * to be used past a roll-over in the next cycle.
54 * When this happens (i.e. task ASID > asid tracker), task needs to refresh
55 * its ASID, aligning it to current value of tracker. If the task doesn't get
56 * scheduled past a roll-over, hence its ASID is not yet realigned with
57 * tracker, such ASID is anyways safely reusable because it is
58 * gauranteed that TLB entries with that ASID wont exist.
59 */ 42 */
60 43
61#define FIRST_ASID 0 44#define MM_CTXT_ASID_MASK 0x000000ff /* MMU PID reg :8 bit PID */
62#define MAX_ASID 255 /* 8 bit PID field in PID Aux reg */ 45#define MM_CTXT_CYCLE_MASK (~MM_CTXT_ASID_MASK)
63#define NO_ASID (MAX_ASID + 1) /* ASID Not alloc to mmu ctxt */ 46
64#define NUM_ASID ((MAX_ASID - FIRST_ASID) + 1) 47#define MM_CTXT_FIRST_CYCLE (MM_CTXT_ASID_MASK + 1)
48#define MM_CTXT_NO_ASID 0UL
65 49
66/* ASID to mm struct mapping */ 50#define hw_pid(mm) (mm->context.asid & MM_CTXT_ASID_MASK)
67extern struct mm_struct *asid_mm_map[NUM_ASID + 1];
68 51
69extern int asid_cache; 52extern unsigned int asid_cache;
70 53
71/* 54/*
72 * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle) 55 * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle)
@@ -74,59 +57,42 @@ extern int asid_cache;
74 */ 57 */
75static inline void get_new_mmu_context(struct mm_struct *mm) 58static inline void get_new_mmu_context(struct mm_struct *mm)
76{ 59{
77 struct mm_struct *prev_owner;
78 unsigned long flags; 60 unsigned long flags;
79 61
80 local_irq_save(flags); 62 local_irq_save(flags);
81 63
82 /* 64 /*
83 * Move to new ASID if it was not from current alloc-cycle/generation. 65 * Move to new ASID if it was not from current alloc-cycle/generation.
66 * This is done by ensuring that the generation bits in both mm->ASID
67 * and cpu's ASID counter are exactly same.
84 * 68 *
85 * Note: Callers needing new ASID unconditionally, independent of 69 * Note: Callers needing new ASID unconditionally, independent of
86 * generation, e.g. local_flush_tlb_mm() for forking parent, 70 * generation, e.g. local_flush_tlb_mm() for forking parent,
87 * first need to destroy the context, setting it to invalid 71 * first need to destroy the context, setting it to invalid
88 * value. 72 * value.
89 */ 73 */
90 if (mm->context.asid <= asid_cache) 74 if (!((mm->context.asid ^ asid_cache) & MM_CTXT_CYCLE_MASK))
91 goto set_hw; 75 goto set_hw;
92 76
93 /* 77 /* move to new ASID and handle rollover */
94 * Relinquish the currently owned ASID (if any). 78 if (unlikely(!(++asid_cache & MM_CTXT_ASID_MASK))) {
95 * Doing unconditionally saves a cmp-n-branch; for already unused
96 * ASID slot, the value was/remains NULL
97 */
98 asid_mm_map[mm->context.asid] = (struct mm_struct *)NULL;
99 79
100 /* move to new ASID */
101 if (++asid_cache > MAX_ASID) { /* ASID roll-over */
102 asid_cache = FIRST_ASID;
103 flush_tlb_all(); 80 flush_tlb_all();
104 }
105 81
106 /* 82 /*
107 * Is next ASID already owned by some-one else (we are stealing it). 83 * Above checke for rollover of 8 bit ASID in 32 bit container.
108 * If so, let the orig owner be aware of this, so when it runs, it 84 * If the container itself wrapped around, set it to a non zero
109 * asks for a brand new ASID. This would only happen for a long-lived 85 * "generation" to distinguish from no context
110 * task with ASID from prev allocation cycle (before ASID roll-over). 86 */
111 * 87 if (!asid_cache)
112 * This might look wrong - if we are re-using some other task's ASID, 88 asid_cache = MM_CTXT_FIRST_CYCLE;
113 * won't we use it's stale TLB entries too. Actually the algorithm takes 89 }
114 * care of such a case: it ensures that task with ASID from prev alloc
115 * cycle, when scheduled will refresh it's ASID
116 * The stealing scenario described here will only happen if that task
117 * didn't get a chance to refresh it's ASID - implying stale entries
118 * won't exist.
119 */
120 prev_owner = asid_mm_map[asid_cache];
121 if (prev_owner)
122 prev_owner->context.asid = NO_ASID;
123 90
124 /* Assign new ASID to tsk */ 91 /* Assign new ASID to tsk */
125 asid_mm_map[asid_cache] = mm;
126 mm->context.asid = asid_cache; 92 mm->context.asid = asid_cache;
127 93
128set_hw: 94set_hw:
129 write_aux_reg(ARC_REG_PID, mm->context.asid | MMU_ENABLE); 95 write_aux_reg(ARC_REG_PID, hw_pid(mm) | MMU_ENABLE);
130 96
131 local_irq_restore(flags); 97 local_irq_restore(flags);
132} 98}
@@ -138,7 +104,7 @@ set_hw:
138static inline int 104static inline int
139init_new_context(struct task_struct *tsk, struct mm_struct *mm) 105init_new_context(struct task_struct *tsk, struct mm_struct *mm)
140{ 106{
141 mm->context.asid = NO_ASID; 107 mm->context.asid = MM_CTXT_NO_ASID;
142 return 0; 108 return 0;
143} 109}
144 110
@@ -167,14 +133,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
167 133
168static inline void destroy_context(struct mm_struct *mm) 134static inline void destroy_context(struct mm_struct *mm)
169{ 135{
170 unsigned long flags; 136 mm->context.asid = MM_CTXT_NO_ASID;
171
172 local_irq_save(flags);
173
174 asid_mm_map[mm->context.asid] = NULL;
175 mm->context.asid = NO_ASID;
176
177 local_irq_restore(flags);
178} 137}
179 138
180/* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping 139/* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping
diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c
index b5c5e0aa0aaa..71cb26df4255 100644
--- a/arch/arc/mm/tlb.c
+++ b/arch/arc/mm/tlb.c
@@ -100,13 +100,7 @@
100 100
101 101
102/* A copy of the ASID from the PID reg is kept in asid_cache */ 102/* A copy of the ASID from the PID reg is kept in asid_cache */
103int asid_cache = FIRST_ASID; 103unsigned int asid_cache = MM_CTXT_FIRST_CYCLE;
104
105/* ASID to mm struct mapping. We have one extra entry corresponding to
106 * NO_ASID to save us a compare when clearing the mm entry for old asid
107 * see get_new_mmu_context (asm-arc/mmu_context.h)
108 */
109struct mm_struct *asid_mm_map[NUM_ASID + 1];
110 104
111/* 105/*
112 * Utility Routine to erase a J-TLB entry 106 * Utility Routine to erase a J-TLB entry
@@ -281,7 +275,6 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
281 unsigned long end) 275 unsigned long end)
282{ 276{
283 unsigned long flags; 277 unsigned long flags;
284 unsigned int asid;
285 278
286 /* If range @start to @end is more than 32 TLB entries deep, 279 /* If range @start to @end is more than 32 TLB entries deep,
287 * its better to move to a new ASID rather than searching for 280 * its better to move to a new ASID rather than searching for
@@ -303,11 +296,10 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
303 start &= PAGE_MASK; 296 start &= PAGE_MASK;
304 297
305 local_irq_save(flags); 298 local_irq_save(flags);
306 asid = vma->vm_mm->context.asid;
307 299
308 if (asid != NO_ASID) { 300 if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
309 while (start < end) { 301 while (start < end) {
310 tlb_entry_erase(start | (asid & 0xff)); 302 tlb_entry_erase(start | hw_pid(vma->vm_mm));
311 start += PAGE_SIZE; 303 start += PAGE_SIZE;
312 } 304 }
313 } 305 }
@@ -361,9 +353,8 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
361 */ 353 */
362 local_irq_save(flags); 354 local_irq_save(flags);
363 355
364 if (vma->vm_mm->context.asid != NO_ASID) { 356 if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
365 tlb_entry_erase((page & PAGE_MASK) | 357 tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm));
366 (vma->vm_mm->context.asid & 0xff));
367 utlb_invalidate(); 358 utlb_invalidate();
368 } 359 }
369 360
@@ -709,7 +700,8 @@ void tlb_paranoid_check(unsigned int mm_asid, unsigned long addr)
709 * - SW needs to have a valid ASID 700 * - SW needs to have a valid ASID
710 */ 701 */
711 if (addr < 0x70000000 && 702 if (addr < 0x70000000 &&
712 ((mmu_asid != mm_asid) || (mm_asid == NO_ASID))) 703 ((mm_asid == MM_CTXT_NO_ASID) ||
704 (mmu_asid != (mm_asid & MM_CTXT_ASID_MASK))))
713 print_asid_mismatch(mm_asid, mmu_asid, 0); 705 print_asid_mismatch(mm_asid, mmu_asid, 0);
714} 706}
715#endif 707#endif
diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S
index 88897a112d55..cf7d7d9ad695 100644
--- a/arch/arc/mm/tlbex.S
+++ b/arch/arc/mm/tlbex.S
@@ -140,12 +140,15 @@ ex_saved_reg1:
140 GET_CURR_TASK_ON_CPU r3 140 GET_CURR_TASK_ON_CPU r3
141 ld r0, [r3, TASK_ACT_MM] 141 ld r0, [r3, TASK_ACT_MM]
142 ld r0, [r0, MM_CTXT+MM_CTXT_ASID] 142 ld r0, [r0, MM_CTXT+MM_CTXT_ASID]
143 breq r0, 0, 55f ; Error if no ASID allocated
143 144
144 lr r1, [ARC_REG_PID] 145 lr r1, [ARC_REG_PID]
145 and r1, r1, 0xFF 146 and r1, r1, 0xFF
146 147
147 breq r1, r0, 5f 148 and r2, r0, 0xFF ; MMU PID bits only for comparison
149 breq r1, r2, 5f
148 150
15155:
149 ; Error if H/w and S/w ASID don't match, but NOT if in kernel mode 152 ; Error if H/w and S/w ASID don't match, but NOT if in kernel mode
150 lr r2, [erstatus] 153 lr r2, [erstatus]
151 bbit0 r2, STATUS_U_BIT, 5f 154 bbit0 r2, STATUS_U_BIT, 5f