1 files changed, 141 insertions, 97 deletions

diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 8d999249d357..37a23c223705 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -15,6 +15,7 @@
 #include <linux/spinlock.h>
 #include <linux/module.h>
 #include <linux/quicklist.h>
+#include <linux/rcupdate.h>
 
 #include <asm/system.h>
 #include <asm/pgtable.h>
@@ -25,30 +26,10 @@
 
 #ifndef CONFIG_64BIT
 #define ALLOC_ORDER     1
-#define TABLES_PER_PAGE 4
-#define FRAG_MASK       15UL
-#define SECOND_HALVES   10UL
-
-void clear_table_pgstes(unsigned long *table)
-{
-        clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/4);
-        memset(table + 256, 0, PAGE_SIZE/4);
-        clear_table(table + 512, _PAGE_TYPE_EMPTY, PAGE_SIZE/4);
-        memset(table + 768, 0, PAGE_SIZE/4);
-}
-
+#define FRAG_MASK       0x0f
 #else
 #define ALLOC_ORDER     2
-#define TABLES_PER_PAGE 2
-#define FRAG_MASK       3UL
-#define SECOND_HALVES   2UL
-
-void clear_table_pgstes(unsigned long *table)
-{
-        clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2);
-        memset(table + 256, 0, PAGE_SIZE/2);
-}
-
+#define FRAG_MASK       0x03
 #endif
 
 unsigned long VMALLOC_START = VMALLOC_END - VMALLOC_SIZE;
@@ -63,37 +44,17 @@ static int __init parse_vmalloc(char *arg)
 }
 early_param("vmalloc", parse_vmalloc);
 
-unsigned long *crst_table_alloc(struct mm_struct *mm, int noexec)
+unsigned long *crst_table_alloc(struct mm_struct *mm)
 {
         struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
 
         if (!page)
                 return NULL;
-        page->index = 0;
-        if (noexec) {
-                struct page *shadow = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
-                if (!shadow) {
-                        __free_pages(page, ALLOC_ORDER);
-                        return NULL;
-                }
-                page->index = page_to_phys(shadow);
-        }
-        spin_lock(&mm->context.list_lock);
-        list_add(&page->lru, &mm->context.crst_list);
-        spin_unlock(&mm->context.list_lock);
         return (unsigned long *) page_to_phys(page);
 }
 
 void crst_table_free(struct mm_struct *mm, unsigned long *table)
 {
-        unsigned long *shadow = get_shadow_table(table);
-        struct page *page = virt_to_page(table);
-
-        spin_lock(&mm->context.list_lock);
-        list_del(&page->lru);
-        spin_unlock(&mm->context.list_lock);
-        if (shadow)
-                free_pages((unsigned long) shadow, ALLOC_ORDER);
         free_pages((unsigned long) table, ALLOC_ORDER);
 }
 
@@ -105,10 +66,10 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long limit)
 
         BUG_ON(limit > (1UL << 53));
 repeat:
-        table = crst_table_alloc(mm, mm->context.noexec);
+        table = crst_table_alloc(mm);
         if (!table)
                 return -ENOMEM;
-        spin_lock(&mm->page_table_lock);
+        spin_lock_bh(&mm->page_table_lock);
         if (mm->context.asce_limit < limit) {
                 pgd = (unsigned long *) mm->pgd;
                 if (mm->context.asce_limit <= (1UL << 31)) {
@@ -130,7 +91,7 @@ repeat:
                 mm->task_size = mm->context.asce_limit;
                 table = NULL;
         }
-        spin_unlock(&mm->page_table_lock);
+        spin_unlock_bh(&mm->page_table_lock);
         if (table)
                 crst_table_free(mm, table);
         if (mm->context.asce_limit < limit)
@@ -172,94 +133,175 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
 }
 #endif
 
+static inline unsigned int atomic_xor_bits(atomic_t *v, unsigned int bits)
+{
+        unsigned int old, new;
+
+        do {
+                old = atomic_read(v);
+                new = old ^ bits;
+        } while (atomic_cmpxchg(v, old, new) != old);
+        return new;
+}
+
 /*
  * page table entry allocation/free routines.
  */
+#ifdef CONFIG_PGSTE
+static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm)
+{
+        struct page *page;
+        unsigned long *table;
+
+        page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
+        if (!page)
+                return NULL;
+        pgtable_page_ctor(page);
+        atomic_set(&page->_mapcount, 3);
+        table = (unsigned long *) page_to_phys(page);
+        clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2);
+        clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2);
+        return table;
+}
+
+static inline void page_table_free_pgste(unsigned long *table)
+{
+        struct page *page;
+
+        page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
+        pgtable_page_ctor(page);
+        atomic_set(&page->_mapcount, -1);
+        __free_page(page);
+}
+#endif
+
 unsigned long *page_table_alloc(struct mm_struct *mm)
 {
         struct page *page;
         unsigned long *table;
-        unsigned long bits;
+        unsigned int mask, bit;
 
-        bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL;
-        spin_lock(&mm->context.list_lock);
-        page = NULL;
+#ifdef CONFIG_PGSTE
+        if (mm_has_pgste(mm))
+                return page_table_alloc_pgste(mm);
+#endif
+        /* Allocate fragments of a 4K page as 1K/2K page table */
+        spin_lock_bh(&mm->context.list_lock);
+        mask = FRAG_MASK;
         if (!list_empty(&mm->context.pgtable_list)) {
                 page = list_first_entry(&mm->context.pgtable_list,
                                         struct page, lru);
-                if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1))
-                        page = NULL;
+                table = (unsigned long *) page_to_phys(page);
+                mask = atomic_read(&page->_mapcount);
+                mask = mask | (mask >> 4);
         }
-        if (!page) {
-                spin_unlock(&mm->context.list_lock);
+        if ((mask & FRAG_MASK) == FRAG_MASK) {
+                spin_unlock_bh(&mm->context.list_lock);
                 page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
                 if (!page)
                         return NULL;
                 pgtable_page_ctor(page);
-                page->flags &= ~FRAG_MASK;
+                atomic_set(&page->_mapcount, 1);
                 table = (unsigned long *) page_to_phys(page);
-                if (mm->context.has_pgste)
-                        clear_table_pgstes(table);
-                else
-                        clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
-                spin_lock(&mm->context.list_lock);
+                clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
+                spin_lock_bh(&mm->context.list_lock);
                 list_add(&page->lru, &mm->context.pgtable_list);
+        } else {
+                for (bit = 1; mask & bit; bit <<= 1)
+                        table += PTRS_PER_PTE;
+                mask = atomic_xor_bits(&page->_mapcount, bit);
+                if ((mask & FRAG_MASK) == FRAG_MASK)
+                        list_del(&page->lru);
         }
-        table = (unsigned long *) page_to_phys(page);
-        while (page->flags & bits) {
-                table += 256;
-                bits <<= 1;
-        }
-        page->flags |= bits;
-        if ((page->flags & FRAG_MASK) == ((1UL << TABLES_PER_PAGE) - 1))
-                list_move_tail(&page->lru, &mm->context.pgtable_list);
-        spin_unlock(&mm->context.list_lock);
+        spin_unlock_bh(&mm->context.list_lock);
         return table;
 }
 
 void page_table_free(struct mm_struct *mm, unsigned long *table)
 {
         struct page *page;
-        unsigned long bits;
+        unsigned int bit, mask;
 
-        bits = (mm->context.noexec || mm->context.has_pgste) ? 3UL : 1UL;
-        bits <<= (__pa(table) & (PAGE_SIZE - 1)) / 256 / sizeof(unsigned long);
+#ifdef CONFIG_PGSTE
+        if (mm_has_pgste(mm))
+                return page_table_free_pgste(table);
+#endif
+        /* Free 1K/2K page table fragment of a 4K page */
         page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
-        spin_lock(&mm->context.list_lock);
-        page->flags ^= bits;
-        if (page->flags & FRAG_MASK) {
-                /* Page now has some free pgtable fragments. */
-                list_move(&page->lru, &mm->context.pgtable_list);
-                page = NULL;
-        } else
-                /* All fragments of the 4K page have been freed. */
+        bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)));
+        spin_lock_bh(&mm->context.list_lock);
+        if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
                 list_del(&page->lru);
-        spin_unlock(&mm->context.list_lock);
-        if (page) {
+        mask = atomic_xor_bits(&page->_mapcount, bit);
+        if (mask & FRAG_MASK)
+                list_add(&page->lru, &mm->context.pgtable_list);
+        spin_unlock_bh(&mm->context.list_lock);
+        if (mask == 0) {
                 pgtable_page_dtor(page);
+                atomic_set(&page->_mapcount, -1);
                 __free_page(page);
         }
 }
 
-void disable_noexec(struct mm_struct *mm, struct task_struct *tsk)
+#ifdef CONFIG_HAVE_RCU_TABLE_FREE
+
+static void __page_table_free_rcu(void *table, unsigned bit)
 {
         struct page *page;
 
-        spin_lock(&mm->context.list_lock);
-        /* Free shadow region and segment tables. */
-        list_for_each_entry(page, &mm->context.crst_list, lru)
-                if (page->index) {
-                        free_pages((unsigned long) page->index, ALLOC_ORDER);
-                        page->index = 0;
-                }
-        /* "Free" second halves of page tables. */
-        list_for_each_entry(page, &mm->context.pgtable_list, lru)
-                page->flags &= ~SECOND_HALVES;
-        spin_unlock(&mm->context.list_lock);
-        mm->context.noexec = 0;
-        update_mm(mm, tsk);
+#ifdef CONFIG_PGSTE
+        if (bit == FRAG_MASK)
+                return page_table_free_pgste(table);
+#endif
+        /* Free 1K/2K page table fragment of a 4K page */
+        page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
+        if (atomic_xor_bits(&page->_mapcount, bit) == 0) {
+                pgtable_page_dtor(page);
+                atomic_set(&page->_mapcount, -1);
+                __free_page(page);
+        }
 }
 
+void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table)
+{
+        struct mm_struct *mm;
+        struct page *page;
+        unsigned int bit, mask;
+
+        mm = tlb->mm;
+#ifdef CONFIG_PGSTE
+        if (mm_has_pgste(mm)) {
+                table = (unsigned long *) (__pa(table) | FRAG_MASK);
+                tlb_remove_table(tlb, table);
+                return;
+        }
+#endif
+        bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)));
+        page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
+        spin_lock_bh(&mm->context.list_lock);
+        if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
+                list_del(&page->lru);
+        mask = atomic_xor_bits(&page->_mapcount, bit | (bit << 4));
+        if (mask & FRAG_MASK)
+                list_add_tail(&page->lru, &mm->context.pgtable_list);
+        spin_unlock_bh(&mm->context.list_lock);
+        table = (unsigned long *) (__pa(table) | (bit << 4));
+        tlb_remove_table(tlb, table);
+}
+
+void __tlb_remove_table(void *_table)
+{
+        void *table = (void *)((unsigned long) _table & PAGE_MASK);
+        unsigned type = (unsigned long) _table & ~PAGE_MASK;
+
+        if (type)
+                __page_table_free_rcu(table, type);
+        else
+                free_pages((unsigned long) table, ALLOC_ORDER);
+}
+
+#endif
+
 /*
  * switch on pgstes for its userspace process (for kvm)
  */
@@ -273,7 +315,7 @@ int s390_enable_sie(void)
                 return -EINVAL;
 
         /* Do we have pgstes? if yes, we are done */
-        if (tsk->mm->context.has_pgste)
+        if (mm_has_pgste(tsk->mm))
                 return 0;
 
         /* lets check if we are allowed to replace the mm */
@@ -312,6 +354,8 @@ int s390_enable_sie(void)
         tsk->mm = tsk->active_mm = mm;
         preempt_disable();
         update_mm(mm, tsk);
+        atomic_inc(&mm->context.attach_count);
+        atomic_dec(&old_mm->context.attach_count);
         cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
         preempt_enable();
         task_unlock(tsk);