s390/mm: pmdp_get_and_clear_full optimization

Analog to ptep_get_and_clear_full define a variant of the
pmpd_get_and_clear primitive which gets the full hint from the
mmu_gather struct. This allows s390 to avoid a costly instruction
when destroying an address space.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

3 files changed, 26 insertions, 1 deletions

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 5ef1a266936a..5e102422c9ab 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -1651,6 +1651,19 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
         return pmd;
 }
 
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR_FULL
+static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm,
+                                            unsigned long address,
+                                            pmd_t *pmdp, int full)
+{
+        pmd_t pmd = *pmdp;
+
+        if (!full)
+                pmdp_flush_lazy(mm, address, pmdp);
+        pmd_clear(pmdp);
+        return pmd;
+}
+
 #define __HAVE_ARCH_PMDP_CLEAR_FLUSH
 static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
                                      unsigned long address, pmd_t *pmdp)
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index 752e30d63904..177d5973b132 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -103,6 +103,17 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 #endif
 
+#ifndef __HAVE_ARCH_PMDP_GET_AND_CLEAR_FULL
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm,
+                                            unsigned long address, pmd_t *pmdp,
+                                            int full)
+{
+        return pmdp_get_and_clear(mm, address, pmdp);
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+#endif
+
 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
                                             unsigned long address, pte_t *ptep,
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 7e9c15cb93a9..6a37f1b2ed1e 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1400,7 +1400,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                  * pgtable_trans_huge_withdraw after finishing pmdp related
                  * operations.
                  */
-                orig_pmd = pmdp_get_and_clear(tlb->mm, addr, pmd);
+                orig_pmd = pmdp_get_and_clear_full(tlb->mm, addr, pmd,
+                                                   tlb->fullmm);
                 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
                 pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
                 if (is_huge_zero_pmd(orig_pmd)) {