mmu_gather: move minimal range calculations into generic code

On architectures with hardware broadcasting of TLB invalidation messages
, it makes sense to reduce the range of the mmu_gather structure when
unmapping page ranges based on the dirty address information passed to
tlb_remove_tlb_entry.

arm64 already does this by directly manipulating the start/end fields
of the gather structure, but this confuses the generic code which
does not expect these fields to change and can end up calculating
invalid, negative ranges when forcing a flush in zap_pte_range.

This patch moves the minimal range calculation out of the arm64 code
and into the generic implementation, simplifying zap_pte_range in the
process (which no longer needs to care about start/end, since they will
point to the appropriate ranges already). With the range being tracked
by core code, the need_flush flag is dropped in favour of checking that
the end of the range has actually been set.

Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Russell King - ARM Linux <linux@arm.linux.org.uk>
Cc: Michal Simek <monstr@monstr.eu>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>

1 files changed, 8 insertions, 22 deletions

diff --git a/mm/memory.c b/mm/memory.c
index 1cc6bfbd872e..c71edae9ba44 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -220,9 +220,6 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long
         /* Is it from 0 to ~0? */
         tlb->fullmm     = !(start | (end+1));
         tlb->need_flush_all = 0;
-        tlb->start      = start;
-        tlb->end        = end;
-        tlb->need_flush = 0;
         tlb->local.next = NULL;
         tlb->local.nr   = 0;
         tlb->local.max  = ARRAY_SIZE(tlb->__pages);
@@ -232,15 +229,20 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
         tlb->batch = NULL;
 #endif
+
+        __tlb_reset_range(tlb);
 }
 
 static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
 {
-        tlb->need_flush = 0;
+        if (!tlb->end)
+                return;
+
         tlb_flush(tlb);
 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
         tlb_table_flush(tlb);
 #endif
+        __tlb_reset_range(tlb);
 }
 
 static void tlb_flush_mmu_free(struct mmu_gather *tlb)
@@ -256,8 +258,6 @@ static void tlb_flush_mmu_free(struct mmu_gather *tlb)
 
 void tlb_flush_mmu(struct mmu_gather *tlb)
 {
-        if (!tlb->need_flush)
-                return;
         tlb_flush_mmu_tlbonly(tlb);
         tlb_flush_mmu_free(tlb);
 }
@@ -292,7 +292,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
         struct mmu_gather_batch *batch;
 
-        VM_BUG_ON(!tlb->need_flush);
+        VM_BUG_ON(!tlb->end);
 
         batch = tlb->active;
         batch->pages[batch->nr++] = page;
@@ -359,8 +359,6 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
 {
         struct mmu_table_batch **batch = &tlb->batch;
 
-        tlb->need_flush = 1;
-
         /*
          * When there's less then two users of this mm there cannot be a
          * concurrent page-table walk.
@@ -1185,20 +1183,8 @@ again:
         arch_leave_lazy_mmu_mode();
 
         /* Do the actual TLB flush before dropping ptl */
-        if (force_flush) {
-                unsigned long old_end;
-
-                /*
-                 * Flush the TLB just for the previous segment,
-                 * then update the range to be the remaining
-                 * TLB range.
-                 */
-                old_end = tlb->end;
-                tlb->end = addr;
+        if (force_flush)
                 tlb_flush_mmu_tlbonly(tlb);
-                tlb->start = addr;
-                tlb->end = old_end;
-        }
         pte_unmap_unlock(start_pte, ptl);
 
         /*