s390/mm: simplify the TLB flushing code

ptep_flush_lazy and pmdp_flush_lazy use mm->context.attach_count to
decide between a lazy TLB flush vs an immediate TLB flush. The field
contains two 16-bit counters, the number of CPUs that have the mm
attached and can create TLB entries for it and the number of CPUs in
the middle of a page table update.

The __tlb_flush_asce, ptep_flush_direct and pmdp_flush_direct functions
use the attach counter and a mask check with mm_cpumask(mm) to decide
between a local flush local of the current CPU and a global flush.

For all these functions the decision between lazy vs immediate and
local vs global TLB flush can be based on CPU masks. There are two
masks:  the mm->context.cpu_attach_mask with the CPUs that are actively
using the mm, and the mm_cpumask(mm) with the CPUs that have used the
mm since the last full flush. The decision between lazy vs immediate
flush is based on the mm->context.cpu_attach_mask, to decide between
local vs global flush the mm_cpumask(mm) is used.

With this patch all checks will use the CPU masks, the old counter
mm->context.attach_count with its two 16-bit values is turned into a
single counter mm->context.flush_count that keeps track of the number
of CPUs with incomplete page table updates. The sole user of this
counter is finish_arch_post_lock_switch() which waits for the end of
all page table updates.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

2 files changed, 15 insertions, 23 deletions

diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 44db60d9e519..de2cdf4fbb9a 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -118,10 +118,8 @@ void mark_rodata_ro(void)
 
 void __init mem_init(void)
 {
-        if (MACHINE_HAS_TLB_LC)
-                cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask);
+        cpumask_set_cpu(0, &init_mm.context.cpu_attach_mask);
         cpumask_set_cpu(0, mm_cpumask(&init_mm));
-        atomic_set(&init_mm.context.attach_count, 1);
 
         set_max_mapnr(max_low_pfn);
         high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 67111ccbb5e0..74f8f2a8a4e8 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -27,40 +27,37 @@
 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
                                       unsigned long addr, pte_t *ptep)
 {
-        int active, count;
         pte_t old;
 
         old = *ptep;
         if (unlikely(pte_val(old) & _PAGE_INVALID))
                 return old;
-        active = (mm == current->active_mm) ? 1 : 0;
-        count = atomic_add_return(0x10000, &mm->context.attach_count);
-        if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
+        atomic_inc(&mm->context.flush_count);
+        if (MACHINE_HAS_TLB_LC &&
             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
                 __ptep_ipte_local(addr, ptep);
         else
                 __ptep_ipte(addr, ptep);
-        atomic_sub(0x10000, &mm->context.attach_count);
+        atomic_dec(&mm->context.flush_count);
         return old;
 }
 
 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
                                     unsigned long addr, pte_t *ptep)
 {
-        int active, count;
         pte_t old;
 
         old = *ptep;
         if (unlikely(pte_val(old) & _PAGE_INVALID))
                 return old;
-        active = (mm == current->active_mm) ? 1 : 0;
-        count = atomic_add_return(0x10000, &mm->context.attach_count);
-        if ((count & 0xffff) <= active) {
+        atomic_inc(&mm->context.flush_count);
+        if (cpumask_equal(&mm->context.cpu_attach_mask,
+                          cpumask_of(smp_processor_id()))) {
                 pte_val(*ptep) |= _PAGE_INVALID;
                 mm->context.flush_mm = 1;
         } else
                 __ptep_ipte(addr, ptep);
-        atomic_sub(0x10000, &mm->context.attach_count);
+        atomic_dec(&mm->context.flush_count);
         return old;
 }
 
@@ -289,7 +286,6 @@ EXPORT_SYMBOL(ptep_modify_prot_commit);
 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
                                       unsigned long addr, pmd_t *pmdp)
 {
-        int active, count;
         pmd_t old;
 
         old = *pmdp;
@@ -299,36 +295,34 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
                 __pmdp_csp(pmdp);
                 return old;
         }
-        active = (mm == current->active_mm) ? 1 : 0;
-        count = atomic_add_return(0x10000, &mm->context.attach_count);
-        if (MACHINE_HAS_TLB_LC && (count & 0xffff) <= active &&
+        atomic_inc(&mm->context.flush_count);
+        if (MACHINE_HAS_TLB_LC &&
             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
                 __pmdp_idte_local(addr, pmdp);
         else
                 __pmdp_idte(addr, pmdp);
-        atomic_sub(0x10000, &mm->context.attach_count);
+        atomic_dec(&mm->context.flush_count);
         return old;
 }
 
 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
                                     unsigned long addr, pmd_t *pmdp)
 {
-        int active, count;
         pmd_t old;
 
         old = *pmdp;
         if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
                 return old;
-        active = (mm == current->active_mm) ? 1 : 0;
-        count = atomic_add_return(0x10000, &mm->context.attach_count);
-        if ((count & 0xffff) <= active) {
+        atomic_inc(&mm->context.flush_count);
+        if (cpumask_equal(&mm->context.cpu_attach_mask,
+                          cpumask_of(smp_processor_id()))) {
                 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
                 mm->context.flush_mm = 1;
         } else if (MACHINE_HAS_IDTE)
                 __pmdp_idte(addr, pmdp);
         else
                 __pmdp_csp(pmdp);
-        atomic_sub(0x10000, &mm->context.attach_count);
+        atomic_dec(&mm->context.flush_count);
         return old;
 }