1 files changed, 72 insertions, 33 deletions

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 66101f6c6d81..50a75d96f939 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -229,6 +229,7 @@ extern unsigned long MODULES_END;
 #define _PAGE_READ      0x010           /* SW pte read bit */
 #define _PAGE_WRITE     0x020           /* SW pte write bit */
 #define _PAGE_SPECIAL   0x040           /* SW associated with special page */
+#define _PAGE_UNUSED    0x080           /* SW bit for pgste usage state */
 #define __HAVE_ARCH_PTE_SPECIAL
 
 /* Set of bits not changed in pte_modify */
@@ -394,6 +395,12 @@ extern unsigned long MODULES_END;
 
 #endif /* CONFIG_64BIT */
 
+/* Guest Page State used for virtualization */
+#define _PGSTE_GPS_ZERO         0x0000000080000000UL
+#define _PGSTE_GPS_USAGE_MASK   0x0000000003000000UL
+#define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL
+#define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL
+
 /*
  * A user page table pointer has the space-switch-event bit, the
  * private-space-control bit and the storage-alteration-event-control
@@ -617,6 +624,14 @@ static inline int pte_none(pte_t pte)
         return pte_val(pte) == _PAGE_INVALID;
 }
 
+static inline int pte_swap(pte_t pte)
+{
+        /* Bit pattern: (pte & 0x603) == 0x402 */
+        return (pte_val(pte) & (_PAGE_INVALID | _PAGE_PROTECT |
+                                _PAGE_TYPE | _PAGE_PRESENT))
+                == (_PAGE_INVALID | _PAGE_TYPE);
+}
+
 static inline int pte_file(pte_t pte)
 {
         /* Bit pattern: (pte & 0x601) == 0x600 */
@@ -823,20 +838,20 @@ unsigned long gmap_translate(unsigned long address, struct gmap *);
 unsigned long __gmap_fault(unsigned long address, struct gmap *);
 unsigned long gmap_fault(unsigned long address, struct gmap *);
 void gmap_discard(unsigned long from, unsigned long to, struct gmap *);
+void __gmap_zap(unsigned long address, struct gmap *);
 
 void gmap_register_ipte_notifier(struct gmap_notifier *);
 void gmap_unregister_ipte_notifier(struct gmap_notifier *);
 int gmap_ipte_notify(struct gmap *, unsigned long start, unsigned long len);
-void gmap_do_ipte_notify(struct mm_struct *, unsigned long addr, pte_t *);
+void gmap_do_ipte_notify(struct mm_struct *, pte_t *);
 
 static inline pgste_t pgste_ipte_notify(struct mm_struct *mm,
-                                        unsigned long addr,
                                         pte_t *ptep, pgste_t pgste)
 {
 #ifdef CONFIG_PGSTE
         if (pgste_val(pgste) & PGSTE_IN_BIT) {
                 pgste_val(pgste) &= ~PGSTE_IN_BIT;
-                gmap_do_ipte_notify(mm, addr, ptep);
+                gmap_do_ipte_notify(mm, ptep);
         }
 #endif
         return pgste;
@@ -854,6 +869,7 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 
         if (mm_has_pgste(mm)) {
                 pgste = pgste_get_lock(ptep);
+                pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
                 pgste_set_key(ptep, pgste, entry);
                 pgste_set_pte(ptep, entry);
                 pgste_set_unlock(ptep, pgste);
@@ -883,6 +899,12 @@ static inline int pte_young(pte_t pte)
         return (pte_val(pte) & _PAGE_YOUNG) != 0;
 }
 
+#define __HAVE_ARCH_PTE_UNUSED
+static inline int pte_unused(pte_t pte)
+{
+        return pte_val(pte) & _PAGE_UNUSED;
+}
+
 /*
  * pgd/pmd/pte modification functions
  */
@@ -1036,30 +1058,41 @@ static inline int ptep_test_and_clear_user_young(struct mm_struct *mm,
 
 static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
 {
-        if (!(pte_val(*ptep) & _PAGE_INVALID)) {
+        unsigned long pto = (unsigned long) ptep;
+
 #ifndef CONFIG_64BIT
-                /* pto must point to the start of the segment table */
-                pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
-#else
-                /* ipte in zarch mode can do the math */
-                pte_t *pto = ptep;
+        /* pto in ESA mode must point to the start of the segment table */
+        pto &= 0x7ffffc00;
 #endif
-                asm volatile(
-                        "       ipte    %2,%3"
-                        : "=m" (*ptep) : "m" (*ptep),
-                          "a" (pto), "a" (address));
-        }
+        /* Invalidation + global TLB flush for the pte */
+        asm volatile(
+                "       ipte    %2,%3"
+                : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address));
+}
+
+static inline void ptep_flush_direct(struct mm_struct *mm,
+                                     unsigned long address, pte_t *ptep)
+{
+        if (pte_val(*ptep) & _PAGE_INVALID)
+                return;
+        __ptep_ipte(address, ptep);
 }
 
 static inline void ptep_flush_lazy(struct mm_struct *mm,
                                    unsigned long address, pte_t *ptep)
 {
-        int active = (mm == current->active_mm) ? 1 : 0;
+        int active, count;
 
-        if (atomic_read(&mm->context.attach_count) > active)
-                __ptep_ipte(address, ptep);
-        else
+        if (pte_val(*ptep) & _PAGE_INVALID)
+                return;
+        active = (mm == current->active_mm) ? 1 : 0;
+        count = atomic_add_return(0x10000, &mm->context.attach_count);
+        if ((count & 0xffff) <= active) {
+                pte_val(*ptep) |= _PAGE_INVALID;
                 mm->context.flush_mm = 1;
+        } else
+                __ptep_ipte(address, ptep);
+        atomic_sub(0x10000, &mm->context.attach_count);
 }
 
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
@@ -1072,11 +1105,11 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
 
         if (mm_has_pgste(vma->vm_mm)) {
                 pgste = pgste_get_lock(ptep);
-                pgste = pgste_ipte_notify(vma->vm_mm, addr, ptep, pgste);
+                pgste = pgste_ipte_notify(vma->vm_mm, ptep, pgste);
         }
 
         pte = *ptep;
-        __ptep_ipte(addr, ptep);
+        ptep_flush_direct(vma->vm_mm, addr, ptep);
         young = pte_young(pte);
         pte = pte_mkold(pte);
 
@@ -1118,7 +1151,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 
         if (mm_has_pgste(mm)) {
                 pgste = pgste_get_lock(ptep);
-                pgste = pgste_ipte_notify(mm, address, ptep, pgste);
+                pgste = pgste_ipte_notify(mm, ptep, pgste);
         }
 
         pte = *ptep;
@@ -1142,12 +1175,11 @@ static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
 
         if (mm_has_pgste(mm)) {
                 pgste = pgste_get_lock(ptep);
-                pgste_ipte_notify(mm, address, ptep, pgste);
+                pgste_ipte_notify(mm, ptep, pgste);
         }
 
         pte = *ptep;
         ptep_flush_lazy(mm, address, ptep);
-        pte_val(*ptep) |= _PAGE_INVALID;
 
         if (mm_has_pgste(mm)) {
                 pgste = pgste_update_all(&pte, pgste);
@@ -1180,14 +1212,17 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
 
         if (mm_has_pgste(vma->vm_mm)) {
                 pgste = pgste_get_lock(ptep);
-                pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
+                pgste = pgste_ipte_notify(vma->vm_mm, ptep, pgste);
         }
 
         pte = *ptep;
-        __ptep_ipte(address, ptep);
+        ptep_flush_direct(vma->vm_mm, address, ptep);
         pte_val(*ptep) = _PAGE_INVALID;
 
         if (mm_has_pgste(vma->vm_mm)) {
+                if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
+                    _PGSTE_GPS_USAGE_UNUSED)
+                        pte_val(pte) |= _PAGE_UNUSED;
                 pgste = pgste_update_all(&pte, pgste);
                 pgste_set_unlock(ptep, pgste);
         }
@@ -1211,7 +1246,7 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 
         if (!full && mm_has_pgste(mm)) {
                 pgste = pgste_get_lock(ptep);
-                pgste = pgste_ipte_notify(mm, address, ptep, pgste);
+                pgste = pgste_ipte_notify(mm, ptep, pgste);
         }
 
         pte = *ptep;
@@ -1236,7 +1271,7 @@ static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,
         if (pte_write(pte)) {
                 if (mm_has_pgste(mm)) {
                         pgste = pgste_get_lock(ptep);
-                        pgste = pgste_ipte_notify(mm, address, ptep, pgste);
+                        pgste = pgste_ipte_notify(mm, ptep, pgste);
                 }
 
                 ptep_flush_lazy(mm, address, ptep);
@@ -1262,10 +1297,10 @@ static inline int ptep_set_access_flags(struct vm_area_struct *vma,
                 return 0;
         if (mm_has_pgste(vma->vm_mm)) {
                 pgste = pgste_get_lock(ptep);
-                pgste = pgste_ipte_notify(vma->vm_mm, address, ptep, pgste);
+                pgste = pgste_ipte_notify(vma->vm_mm, ptep, pgste);
         }
 
-        __ptep_ipte(address, ptep);
+        ptep_flush_direct(vma->vm_mm, address, ptep);
 
         if (mm_has_pgste(vma->vm_mm)) {
                 pgste_set_pte(ptep, entry);
@@ -1449,12 +1484,16 @@ static inline pmd_t pmd_mkwrite(pmd_t pmd)
 static inline void pmdp_flush_lazy(struct mm_struct *mm,
                                    unsigned long address, pmd_t *pmdp)
 {
-        int active = (mm == current->active_mm) ? 1 : 0;
+        int active, count;
 
-        if ((atomic_read(&mm->context.attach_count) & 0xffff) > active)
-                __pmd_idte(address, pmdp);
-        else
+        active = (mm == current->active_mm) ? 1 : 0;
+        count = atomic_add_return(0x10000, &mm->context.attach_count);
+        if ((count & 0xffff) <= active) {
+                pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
                 mm->context.flush_mm = 1;
+        } else
+                __pmd_idte(address, pmdp);
+        atomic_sub(0x10000, &mm->context.attach_count);
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE