ARC: [ASID] Track ASID allocation cycles/generations

This helps remove asid-to-mm reverse map

While mm->context.id contains the ASID assigned to a process, our ASID
allocator also used asid_mm_map[] reverse map. In a new allocation
cycle (mm->ASID >= @asid_cache), the Round Robin ASID allocator used this
to check if new @asid_cache belonged to some mm2 (from prev cycle).
If so, it could locate that mm using the ASID reverse map, and mark that
mm as unallocated ASID, to force it to refresh at the time of switch_mm()

However, for SMP, the reverse map has to be maintained per CPU, so
becomes 2 dimensional, hence got rid of it.

With reverse map gone, it is NOT possible to reach out to current
assignee. So we track the ASID allocation generation/cycle and
on every switch_mm(), check if the current generation of CPU ASID is
same as mm's ASID; If not it is refreshed.

(Based loosely on arch/sh implementation)

Signed-off-by: Vineet Gupta <vgupta@synopsys.com>

4 files changed, 40 insertions, 86 deletions

diff --git a/arch/arc/include/asm/mmu.h b/arch/arc/include/asm/mmu.h
index 1639f25e47b1..c82db8bd7270 100644
--- a/arch/arc/include/asm/mmu.h
+++ b/arch/arc/include/asm/mmu.h
@@ -48,7 +48,7 @@
 #ifndef __ASSEMBLY__
 
 typedef struct {
-        unsigned long asid;     /* Pvt Addr-Space ID for mm */
+        unsigned long asid;     /* 8 bit MMU PID + Generation cycle */
 } mm_context_t;
 
 #ifdef CONFIG_ARC_DBG_TLB_PARANOIA
diff --git a/arch/arc/include/asm/mmu_context.h b/arch/arc/include/asm/mmu_context.h
index 9b09d18f01b3..43a1b51bb8cc 100644
--- a/arch/arc/include/asm/mmu_context.h
+++ b/arch/arc/include/asm/mmu_context.h
@@ -34,39 +34,22 @@
  * When it reaches max 255, the allocation cycle starts afresh by flushing
  * the entire TLB and wrapping ASID back to zero.
  *
- * For book-keeping, Linux uses a couple of data-structures:
- *  -mm_struct has an @asid field to keep a note of task's ASID (needed at the
- *   time of say switch_mm( )
- *  -An array of mm structs @asid_mm_map[] for asid->mm the reverse mapping,
- *  given an ASID, finding the mm struct associated.
- *
- * The round-robin allocation algorithm allows for ASID stealing.
- * If asid tracker is at "x-1", a new req will allocate "x", even if "x" was
- * already assigned to another (switched-out) task. Obviously the prev owner
- * is marked with an invalid ASID to make it request for a new ASID when it
- * gets scheduled next time. However its TLB entries (with ASID "x") could
- * exist, which must be cleared before the same ASID is used by the new owner.
- * Flushing them would be plausible but costly solution. Instead we force a
- * allocation policy quirk, which ensures that a stolen ASID won't have any
- * TLB entries associates, alleviating the need to flush.
- * The quirk essentially is not allowing ASID allocated in prev cycle
- * to be used past a roll-over in the next cycle.
- * When this happens (i.e. task ASID > asid tracker), task needs to refresh
- * its ASID, aligning it to current value of tracker. If the task doesn't get
- * scheduled past a roll-over, hence its ASID is not yet realigned with
- * tracker, such ASID is anyways safely reusable because it is
- * gauranteed that TLB entries with that ASID wont exist.
+ * A new allocation cycle, post rollover, could potentially reassign an ASID
+ * to a different task. Thus the rule is to refresh the ASID in a new cycle.
+ * The 32 bit @asid_cache (and mm->asid) have 8 bits MMU PID and rest 24 bits
+ * serve as cycle/generation indicator and natural 32 bit unsigned math
+ * automagically increments the generation when lower 8 bits rollover.
  */
 
-#define FIRST_ASID  0
-#define MAX_ASID    255                 /* 8 bit PID field in PID Aux reg */
-#define NO_ASID     (MAX_ASID + 1)      /* ASID Not alloc to mmu ctxt */
-#define NUM_ASID    ((MAX_ASID - FIRST_ASID) + 1)
+#define MM_CTXT_ASID_MASK       0x000000ff /* MMU PID reg :8 bit PID */
+#define MM_CTXT_CYCLE_MASK      (~MM_CTXT_ASID_MASK)
+
+#define MM_CTXT_FIRST_CYCLE     (MM_CTXT_ASID_MASK + 1)
+#define MM_CTXT_NO_ASID         0UL
 
-/* ASID to mm struct mapping */
-extern struct mm_struct *asid_mm_map[NUM_ASID + 1];
+#define hw_pid(mm)              (mm->context.asid & MM_CTXT_ASID_MASK)
 
-extern int asid_cache;
+extern unsigned int asid_cache;
 
 /*
  * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle)
@@ -74,59 +57,42 @@ extern int asid_cache;
  */
 static inline void get_new_mmu_context(struct mm_struct *mm)
 {
-        struct mm_struct *prev_owner;
         unsigned long flags;
 
         local_irq_save(flags);
 
         /*
          * Move to new ASID if it was not from current alloc-cycle/generation.
+         * This is done by ensuring that the generation bits in both mm->ASID
+         * and cpu's ASID counter are exactly same.
          *
          * Note: Callers needing new ASID unconditionally, independent of
          *       generation, e.g. local_flush_tlb_mm() for forking  parent,
          *       first need to destroy the context, setting it to invalid
          *       value.
          */
-        if (mm->context.asid <= asid_cache)
+        if (!((mm->context.asid ^ asid_cache) & MM_CTXT_CYCLE_MASK))
                 goto set_hw;
 
-        /*
-         * Relinquish the currently owned ASID (if any).
-         * Doing unconditionally saves a cmp-n-branch; for already unused
-         * ASID slot, the value was/remains NULL
-         */
-        asid_mm_map[mm->context.asid] = (struct mm_struct *)NULL;
+        /* move to new ASID and handle rollover */
+        if (unlikely(!(++asid_cache & MM_CTXT_ASID_MASK))) {
 
-        /* move to new ASID */
-        if (++asid_cache > MAX_ASID) {  /* ASID roll-over */
-                asid_cache = FIRST_ASID;
                 flush_tlb_all();
-        }
 
-        /*
-         * Is next ASID already owned by some-one else (we are stealing it).
-         * If so, let the orig owner be aware of this, so when it runs, it
-         * asks for a brand new ASID. This would only happen for a long-lived
-         * task with ASID from prev allocation cycle (before ASID roll-over).
-         *
-         * This might look wrong - if we are re-using some other task's ASID,
-         * won't we use it's stale TLB entries too. Actually the algorithm takes
-         * care of such a case: it ensures that task with ASID from prev alloc
-         * cycle, when scheduled will refresh it's ASID
-         * The stealing scenario described here will only happen if that task
-         * didn't get a chance to refresh it's ASID - implying stale entries
-         * won't exist.
-         */
-        prev_owner = asid_mm_map[asid_cache];
-        if (prev_owner)
-                prev_owner->context.asid = NO_ASID;
+                /*
+                 * Above checke for rollover of 8 bit ASID in 32 bit container.
+                 * If the container itself wrapped around, set it to a non zero
+                 * "generation" to distinguish from no context
+                 */
+                if (!asid_cache)
+                        asid_cache = MM_CTXT_FIRST_CYCLE;
+        }
 
         /* Assign new ASID to tsk */
-        asid_mm_map[asid_cache] = mm;
         mm->context.asid = asid_cache;
 
 set_hw:
-        write_aux_reg(ARC_REG_PID, mm->context.asid | MMU_ENABLE);
+        write_aux_reg(ARC_REG_PID, hw_pid(mm) | MMU_ENABLE);
 
         local_irq_restore(flags);
 }
@@ -138,7 +104,7 @@ set_hw:
 static inline int
 init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
-        mm->context.asid = NO_ASID;
+        mm->context.asid = MM_CTXT_NO_ASID;
         return 0;
 }
 
@@ -167,14 +133,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 
 static inline void destroy_context(struct mm_struct *mm)
 {
-        unsigned long flags;
-
-        local_irq_save(flags);
-
-        asid_mm_map[mm->context.asid] = NULL;
-        mm->context.asid = NO_ASID;
-
-        local_irq_restore(flags);
+        mm->context.asid = MM_CTXT_NO_ASID;
 }
 
 /* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping
diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c
index b5c5e0aa0aaa..71cb26df4255 100644
--- a/arch/arc/mm/tlb.c
+++ b/arch/arc/mm/tlb.c
@@ -100,13 +100,7 @@
 
 
 /* A copy of the ASID from the PID reg is kept in asid_cache */
-int asid_cache = FIRST_ASID;
-
-/* ASID to mm struct mapping. We have one extra entry corresponding to
- * NO_ASID to save us a compare when clearing the mm entry for old asid
- * see get_new_mmu_context (asm-arc/mmu_context.h)
- */
-struct mm_struct *asid_mm_map[NUM_ASID + 1];
+unsigned int asid_cache = MM_CTXT_FIRST_CYCLE;
 
 /*
  * Utility Routine to erase a J-TLB entry
@@ -281,7 +275,6 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
                            unsigned long end)
 {
         unsigned long flags;
-        unsigned int asid;
 
         /* If range @start to @end is more than 32 TLB entries deep,
          * its better to move to a new ASID rather than searching for
@@ -303,11 +296,10 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
         start &= PAGE_MASK;
 
         local_irq_save(flags);
-        asid = vma->vm_mm->context.asid;
 
-        if (asid != NO_ASID) {
+        if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
                 while (start < end) {
-                        tlb_entry_erase(start | (asid & 0xff));
+                        tlb_entry_erase(start | hw_pid(vma->vm_mm));
                         start += PAGE_SIZE;
                 }
         }
@@ -361,9 +353,8 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
          */
         local_irq_save(flags);
 
-        if (vma->vm_mm->context.asid != NO_ASID) {
-                tlb_entry_erase((page & PAGE_MASK) |
-                                (vma->vm_mm->context.asid & 0xff));
+        if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
+                tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm));
                 utlb_invalidate();
         }
 
@@ -709,7 +700,8 @@ void tlb_paranoid_check(unsigned int mm_asid, unsigned long addr)
          *   - SW needs to have a valid ASID
          */
         if (addr < 0x70000000 &&
-            ((mmu_asid != mm_asid) || (mm_asid == NO_ASID)))
+            ((mm_asid == MM_CTXT_NO_ASID) ||
+              (mmu_asid != (mm_asid & MM_CTXT_ASID_MASK))))
                 print_asid_mismatch(mm_asid, mmu_asid, 0);
 }
 #endif
diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S
index 88897a112d55..cf7d7d9ad695 100644
--- a/arch/arc/mm/tlbex.S
+++ b/arch/arc/mm/tlbex.S
@@ -140,12 +140,15 @@ ex_saved_reg1:
         GET_CURR_TASK_ON_CPU  r3
         ld r0, [r3, TASK_ACT_MM]
         ld r0, [r0, MM_CTXT+MM_CTXT_ASID]
+        breq r0, 0, 55f ; Error if no ASID allocated
 
         lr r1, [ARC_REG_PID]
         and r1, r1, 0xFF
 
-        breq r1, r0, 5f
+        and r2, r0, 0xFF        ; MMU PID bits only for comparison
+        breq r1, r2, 5f
 
+55:
         ; Error if H/w and S/w ASID don't match, but NOT if in kernel mode
         lr  r2, [erstatus]
         bbit0 r2, STATUS_U_BIT, 5f