5 files changed, 123 insertions, 44 deletions

diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h
index 5fb97e19d81e..31977c8dd942 100644
--- a/arch/sparc/include/asm/mmu_64.h
+++ b/arch/sparc/include/asm/mmu_64.h
@@ -93,6 +93,7 @@ typedef struct {
         spinlock_t              lock;
         unsigned long           sparc64_ctx_val;
         unsigned long           huge_pte_count;
+        struct page             *pgtable_page;
         struct tsb_config       tsb_block[MM_NUM_TSBS];
         struct hv_tsb_descr     tsb_descr[MM_NUM_TSBS];
 } mm_context_t;
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index b2df9b8b2d46..087a5c505c69 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -86,7 +86,7 @@ typedef unsigned long pgprot_t;
 
 #endif /* (STRICT_MM_TYPECHECKS) */
 
-typedef struct page *pgtable_t;
+typedef pte_t *pgtable_t;
 
 #define TASK_UNMAPPED_BASE      (test_thread_flag(TIF_32BIT) ? \
                                  (_AC(0x0000000070000000,UL)) : \
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 40b2d7a7023d..0ebca93ef0f5 100644
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -38,51 +38,20 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
         kmem_cache_free(pgtable_cache, pmd);
 }
 
-static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
-                                          unsigned long address)
-{
-        return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
-}
-
-static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
-                                        unsigned long address)
-{
-        struct page *page;
-        pte_t *pte;
-
-        pte = pte_alloc_one_kernel(mm, address);
-        if (!pte)
-                return NULL;
-        page = virt_to_page(pte);
-        pgtable_page_ctor(page);
-        return page;
-}
-
-static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
-{
-        free_page((unsigned long)pte);
-}
-
-static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
-{
-        pgtable_page_dtor(ptepage);
-        __free_page(ptepage);
-}
+extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                                   unsigned long address);
+extern pgtable_t pte_alloc_one(struct mm_struct *mm,
+                               unsigned long address);
+extern void pte_free_kernel(struct mm_struct *mm, pte_t *pte);
+extern void pte_free(struct mm_struct *mm, pgtable_t ptepage);
 
 #define pmd_populate_kernel(MM, PMD, PTE)       pmd_set(PMD, PTE)
-#define pmd_populate(MM,PMD,PTE_PAGE)           \
-        pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE))
-#define pmd_pgtable(pmd) pmd_page(pmd)
+#define pmd_populate(MM, PMD, PTE)              pmd_set(PMD, PTE)
+#define pmd_pgtable(PMD)                        ((pte_t *)__pmd_page(PMD))
 
 #define check_pgt_cache()       do { } while (0)
 
-static inline void pgtable_free(void *table, bool is_page)
-{
-        if (is_page)
-                free_page((unsigned long)table);
-        else
-                kmem_cache_free(pgtable_cache, table);
-}
+extern void pgtable_free(void *table, bool is_page);
 
 #ifdef CONFIG_SMP
 
@@ -113,11 +82,10 @@ static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is
 }
 #endif /* !CONFIG_SMP */
 
-static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage,
+static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte,
                                   unsigned long address)
 {
-        pgtable_page_dtor(ptepage);
-        pgtable_free_tlb(tlb, page_address(ptepage), true);
+        pgtable_free_tlb(tlb, pte, true);
 }
 
 #define __pmd_free_tlb(tlb, pmd, addr)                \
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 809eecf6c797..12ef4ea60c88 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2467,3 +2467,104 @@ void __flush_tlb_all(void)
         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
                              : : "r" (pstate));
 }
+
+static pte_t *get_from_cache(struct mm_struct *mm)
+{
+        struct page *page;
+        pte_t *ret;
+
+        spin_lock(&mm->page_table_lock);
+        page = mm->context.pgtable_page;
+        ret = NULL;
+        if (page) {
+                void *p = page_address(page);
+
+                mm->context.pgtable_page = NULL;
+
+                ret = (pte_t *) (p + (PAGE_SIZE / 2));
+        }
+        spin_unlock(&mm->page_table_lock);
+
+        return ret;
+}
+
+static struct page *__alloc_for_cache(struct mm_struct *mm)
+{
+        struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+                                       __GFP_REPEAT | __GFP_ZERO);
+
+        if (page) {
+                spin_lock(&mm->page_table_lock);
+                if (!mm->context.pgtable_page) {
+                        atomic_set(&page->_count, 2);
+                        mm->context.pgtable_page = page;
+                }
+                spin_unlock(&mm->page_table_lock);
+        }
+        return page;
+}
+
+pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                            unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        page = __alloc_for_cache(mm);
+        if (page)
+                pte = (pte_t *) page_address(page);
+
+        return pte;
+}
+
+pgtable_t pte_alloc_one(struct mm_struct *mm,
+                        unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = get_from_cache(mm);
+        if (pte)
+                return pte;
+
+        page = __alloc_for_cache(mm);
+        if (page) {
+                pgtable_page_ctor(page);
+                pte = (pte_t *) page_address(page);
+        }
+
+        return pte;
+}
+
+void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+        struct page *page = virt_to_page(pte);
+        if (put_page_testzero(page))
+                free_hot_cold_page(page, 0);
+}
+
+static void __pte_free(pgtable_t pte)
+{
+        struct page *page = virt_to_page(pte);
+        if (put_page_testzero(page)) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+}
+
+void pte_free(struct mm_struct *mm, pgtable_t pte)
+{
+        __pte_free(pte);
+}
+
+void pgtable_free(void *table, bool is_page)
+{
+        if (is_page)
+                __pte_free(table);
+        else
+                kmem_cache_free(pgtable_cache, table);
+}
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 70e50ea2a5b6..a35ee832baf3 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -445,6 +445,8 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
         mm->context.huge_pte_count = 0;
 #endif
 
+        mm->context.pgtable_page = NULL;
+
         /* copy_mm() copies over the parent's mm_struct before calling
          * us, so we need to zero out the TSB pointer or else tsb_grow()
          * will be confused and think there is an older TSB to free up.
@@ -483,10 +485,17 @@ static void tsb_destroy_one(struct tsb_config *tp)
 void destroy_context(struct mm_struct *mm)
 {
         unsigned long flags, i;
+        struct page *page;
 
         for (i = 0; i < MM_NUM_TSBS; i++)
                 tsb_destroy_one(&mm->context.tsb_block[i]);
 
+        page = mm->context.pgtable_page;
+        if (page && put_page_testzero(page)) {
+                pgtable_page_dtor(page);
+                free_hot_cold_page(page, 0);
+        }
+
         spin_lock_irqsave(&ctx_alloc_lock, flags);
 
         if (CTX_VALID(mm->context)) {