1 files changed, 91 insertions, 69 deletions

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 2accf7113d13..bd954e96f51c 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -454,12 +454,11 @@ unsigned long gmap_translate(unsigned long address, struct gmap *gmap)
 }
 EXPORT_SYMBOL_GPL(gmap_translate);
 
-/*
- * this function is assumed to be called with mmap_sem held
- */
-unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
+static int gmap_connect_pgtable(unsigned long segment,
+                                unsigned long *segment_ptr,
+                                struct gmap *gmap)
 {
-        unsigned long *segment_ptr, vmaddr, segment;
+        unsigned long vmaddr;
         struct vm_area_struct *vma;
         struct gmap_pgtable *mp;
         struct gmap_rmap *rmap;
@@ -469,48 +468,94 @@ unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
         pud_t *pud;
         pmd_t *pmd;
 
+        mm = gmap->mm;
+        vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
+        vma = find_vma(mm, vmaddr);
+        if (!vma || vma->vm_start > vmaddr)
+                return -EFAULT;
+        /* Walk the parent mm page table */
+        pgd = pgd_offset(mm, vmaddr);
+        pud = pud_alloc(mm, pgd, vmaddr);
+        if (!pud)
+                return -ENOMEM;
+        pmd = pmd_alloc(mm, pud, vmaddr);
+        if (!pmd)
+                return -ENOMEM;
+        if (!pmd_present(*pmd) &&
+            __pte_alloc(mm, vma, pmd, vmaddr))
+                return -ENOMEM;
+        /* pmd now points to a valid segment table entry. */
+        rmap = kmalloc(sizeof(*rmap), GFP_KERNEL|__GFP_REPEAT);
+        if (!rmap)
+                return -ENOMEM;
+        /* Link gmap segment table entry location to page table. */
+        page = pmd_page(*pmd);
+        mp = (struct gmap_pgtable *) page->index;
+        rmap->entry = segment_ptr;
+        spin_lock(&mm->page_table_lock);
+        if (*segment_ptr == segment) {
+                list_add(&rmap->list, &mp->mapper);
+                /* Set gmap segment table entry to page table. */
+                *segment_ptr = pmd_val(*pmd) & PAGE_MASK;
+                rmap = NULL;
+        }
+        spin_unlock(&mm->page_table_lock);
+        kfree(rmap);
+        return 0;
+}
+
+static void gmap_disconnect_pgtable(struct mm_struct *mm, unsigned long *table)
+{
+        struct gmap_rmap *rmap, *next;
+        struct gmap_pgtable *mp;
+        struct page *page;
+        int flush;
+
+        flush = 0;
+        spin_lock(&mm->page_table_lock);
+        page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
+        mp = (struct gmap_pgtable *) page->index;
+        list_for_each_entry_safe(rmap, next, &mp->mapper, list) {
+                *rmap->entry =
+                        _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr;
+                list_del(&rmap->list);
+                kfree(rmap);
+                flush = 1;
+        }
+        spin_unlock(&mm->page_table_lock);
+        if (flush)
+                __tlb_flush_global();
+}
+
+/*
+ * this function is assumed to be called with mmap_sem held
+ */
+unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
+{
+        unsigned long *segment_ptr, segment;
+        struct gmap_pgtable *mp;
+        struct page *page;
+        int rc;
+
         current->thread.gmap_addr = address;
         segment_ptr = gmap_table_walk(address, gmap);
         if (IS_ERR(segment_ptr))
                 return -EFAULT;
         /* Convert the gmap address to an mm address. */
-        segment = *segment_ptr;
-        if (!(segment & _SEGMENT_ENTRY_INV)) {
-                page = pfn_to_page(segment >> PAGE_SHIFT);
-                mp = (struct gmap_pgtable *) page->index;
-                return mp->vmaddr | (address & ~PMD_MASK);
-        } else if (segment & _SEGMENT_ENTRY_RO) {
-                mm = gmap->mm;
-                vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
-                vma = find_vma(mm, vmaddr);
-                if (!vma || vma->vm_start > vmaddr)
-                        return -EFAULT;
-
-                /* Walk the parent mm page table */
-                pgd = pgd_offset(mm, vmaddr);
-                pud = pud_alloc(mm, pgd, vmaddr);
-                if (!pud)
-                        return -ENOMEM;
-                pmd = pmd_alloc(mm, pud, vmaddr);
-                if (!pmd)
-                        return -ENOMEM;
-                if (!pmd_present(*pmd) &&
-                    __pte_alloc(mm, vma, pmd, vmaddr))
-                        return -ENOMEM;
-                /* pmd now points to a valid segment table entry. */
-                rmap = kmalloc(sizeof(*rmap), GFP_KERNEL|__GFP_REPEAT);
-                if (!rmap)
-                        return -ENOMEM;
-                /* Link gmap segment table entry location to page table. */
-                page = pmd_page(*pmd);
-                mp = (struct gmap_pgtable *) page->index;
-                rmap->entry = segment_ptr;
-                spin_lock(&mm->page_table_lock);
-                list_add(&rmap->list, &mp->mapper);
-                spin_unlock(&mm->page_table_lock);
-                /* Set gmap segment table entry to page table. */
-                *segment_ptr = pmd_val(*pmd) & PAGE_MASK;
-                return vmaddr | (address & ~PMD_MASK);
+        while (1) {
+                segment = *segment_ptr;
+                if (!(segment & _SEGMENT_ENTRY_INV)) {
+                        /* Page table is present */
+                        page = pfn_to_page(segment >> PAGE_SHIFT);
+                        mp = (struct gmap_pgtable *) page->index;
+                        return mp->vmaddr | (address & ~PMD_MASK);
+                }
+                if (!(segment & _SEGMENT_ENTRY_RO))
+                        /* Nothing mapped in the gmap address space. */
+                        break;
+                rc = gmap_connect_pgtable(segment, segment_ptr, gmap);
+                if (rc)
+                        return rc;
         }
         return -EFAULT;
 }
@@ -574,29 +619,6 @@ void gmap_discard(unsigned long from, unsigned long to, struct gmap *gmap)
 }
 EXPORT_SYMBOL_GPL(gmap_discard);
 
-void gmap_unmap_notifier(struct mm_struct *mm, unsigned long *table)
-{
-        struct gmap_rmap *rmap, *next;
-        struct gmap_pgtable *mp;
-        struct page *page;
-        int flush;
-
-        flush = 0;
-        spin_lock(&mm->page_table_lock);
-        page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
-        mp = (struct gmap_pgtable *) page->index;
-        list_for_each_entry_safe(rmap, next, &mp->mapper, list) {
-                *rmap->entry =
-                        _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr;
-                list_del(&rmap->list);
-                kfree(rmap);
-                flush = 1;
-        }
-        spin_unlock(&mm->page_table_lock);
-        if (flush)
-                __tlb_flush_global();
-}
-
 static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
                                                     unsigned long vmaddr)
 {
@@ -649,8 +671,8 @@ static inline void page_table_free_pgste(unsigned long *table)
 {
 }
 
-static inline void gmap_unmap_notifier(struct mm_struct *mm,
-                                          unsigned long *table)
+static inline void gmap_disconnect_pgtable(struct mm_struct *mm,
+                                           unsigned long *table)
 {
 }
 
@@ -716,7 +738,7 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
         unsigned int bit, mask;
 
         if (mm_has_pgste(mm)) {
-                gmap_unmap_notifier(mm, table);
+                gmap_disconnect_pgtable(mm, table);
                 return page_table_free_pgste(table);
         }
         /* Free 1K/2K page table fragment of a 4K page */
@@ -759,7 +781,7 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table)
 
         mm = tlb->mm;
         if (mm_has_pgste(mm)) {
-                gmap_unmap_notifier(mm, table);
+                gmap_disconnect_pgtable(mm, table);
                 table = (unsigned long *) (__pa(table) | FRAG_MASK);
                 tlb_remove_table(tlb, table);
                 return;