3 files changed, 189 insertions, 59 deletions

diff --git a/arch/powerpc/include/asm/pgalloc-32.h b/arch/powerpc/include/asm/pgalloc-32.h
index 580cf73b96e8..27b2386f738a 100644
--- a/arch/powerpc/include/asm/pgalloc-32.h
+++ b/arch/powerpc/include/asm/pgalloc-32.h
@@ -37,6 +37,17 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
 extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr);
 extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr);
 
+static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+        free_page((unsigned long)pte);
+}
+
+static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
+{
+        pgtable_page_dtor(ptepage);
+        __free_page(ptepage);
+}
+
 static inline void pgtable_free(void *table, unsigned index_size)
 {
         BUG_ON(index_size); /* 32-bit doesn't use this */
@@ -45,4 +56,38 @@ static inline void pgtable_free(void *table, unsigned index_size)
 
 #define check_pgt_cache()       do { } while (0)
 
+#ifdef CONFIG_SMP
+static inline void pgtable_free_tlb(struct mmu_gather *tlb,
+                                    void *table, int shift)
+{
+        unsigned long pgf = (unsigned long)table;
+        BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+        pgf |= shift;
+        tlb_remove_table(tlb, (void *)pgf);
+}
+
+static inline void __tlb_remove_table(void *_table)
+{
+        void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+        unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+
+        pgtable_free(table, shift);
+}
+#else
+static inline void pgtable_free_tlb(struct mmu_gather *tlb,
+                                    void *table, int shift)
+{
+        pgtable_free(table, shift);
+}
+#endif
+
+static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
+                                  unsigned long address)
+{
+        struct page *page = page_address(table);
+
+        tlb_flush_pgtable(tlb, address);
+        pgtable_page_dtor(page);
+        pgtable_free_tlb(tlb, page, 0);
+}
 #endif /* _ASM_POWERPC_PGALLOC_32_H */
diff --git a/arch/powerpc/include/asm/pgalloc-64.h b/arch/powerpc/include/asm/pgalloc-64.h
index 69e352a5252b..d39012352f94 100644
--- a/arch/powerpc/include/asm/pgalloc-64.h
+++ b/arch/powerpc/include/asm/pgalloc-64.h
@@ -75,8 +75,83 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))
 #define pmd_pgtable(pmd) pmd_page(pmd)
 
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                                          unsigned long address)
+{
+        return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
+}
+
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
+                                      unsigned long address)
+{
+        struct page *page;
+        pte_t *pte;
+
+        pte = pte_alloc_one_kernel(mm, address);
+        if (!pte)
+                return NULL;
+        page = virt_to_page(pte);
+        pgtable_page_ctor(page);
+        return page;
+}
+
+static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+        free_page((unsigned long)pte);
+}
+
+static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
+{
+        pgtable_page_dtor(ptepage);
+        __free_page(ptepage);
+}
+
+static inline void pgtable_free(void *table, unsigned index_size)
+{
+        if (!index_size)
+                free_page((unsigned long)table);
+        else {
+                BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
+                kmem_cache_free(PGT_CACHE(index_size), table);
+        }
+}
+
+#ifdef CONFIG_SMP
+static inline void pgtable_free_tlb(struct mmu_gather *tlb,
+                                    void *table, int shift)
+{
+        unsigned long pgf = (unsigned long)table;
+        BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+        pgf |= shift;
+        tlb_remove_table(tlb, (void *)pgf);
+}
+
+static inline void __tlb_remove_table(void *_table)
+{
+        void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+        unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+
+        pgtable_free(table, shift);
+}
+#else /* !CONFIG_SMP */
+static inline void pgtable_free_tlb(struct mmu_gather *tlb,
+                                    void *table, int shift)
+{
+        pgtable_free(table, shift);
+}
+#endif /* CONFIG_SMP */
+
+static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
+                                  unsigned long address)
+{
+        struct page *page = page_address(table);
+
+        tlb_flush_pgtable(tlb, address);
+        pgtable_page_dtor(page);
+        pgtable_free_tlb(tlb, page, 0);
+}
 
-#else /* CONFIG_PPC_64K_PAGES */
+#else /* if CONFIG_PPC_64K_PAGES */
 
 #define pud_populate(mm, pud, pmd)      pud_set(pud, (unsigned long)pmd)
 
@@ -86,31 +161,25 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
         pmd_set(pmd, (unsigned long)pte);
 }
 
-#define pmd_populate(mm, pmd, pte_page) \
-        pmd_populate_kernel(mm, pmd, page_address(pte_page))
-#define pmd_pgtable(pmd) pmd_page(pmd)
-
-#endif /* CONFIG_PPC_64K_PAGES */
-
-static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
+                                pgtable_t pte_page)
 {
-        return kmem_cache_alloc(PGT_CACHE(PMD_INDEX_SIZE),
-                                GFP_KERNEL|__GFP_REPEAT);
+        pmd_populate_kernel(mm, pmd, page_address(pte_page));
 }
 
-static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+static inline pgtable_t pmd_pgtable(pmd_t pmd)
 {
-        kmem_cache_free(PGT_CACHE(PMD_INDEX_SIZE), pmd);
+        return pmd_page(pmd);
 }
 
 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
                                           unsigned long address)
 {
-        return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
+        return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
 }
 
 static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
-                                        unsigned long address)
+                                      unsigned long address)
 {
         struct page *page;
         pte_t *pte;
@@ -123,6 +192,17 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
         return page;
 }
 
+static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+        free_page((unsigned long)pte);
+}
+
+static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
+{
+        pgtable_page_dtor(ptepage);
+        __free_page(ptepage);
+}
+
 static inline void pgtable_free(void *table, unsigned index_size)
 {
         if (!index_size)
@@ -133,6 +213,55 @@ static inline void pgtable_free(void *table, unsigned index_size)
         }
 }
 
+#ifdef CONFIG_SMP
+static inline void pgtable_free_tlb(struct mmu_gather *tlb,
+                                    void *table, int shift)
+{
+        unsigned long pgf = (unsigned long)table;
+        BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+        pgf |= shift;
+        tlb_remove_table(tlb, (void *)pgf);
+}
+
+static inline void __tlb_remove_table(void *_table)
+{
+        void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+        unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+
+        pgtable_free(table, shift);
+}
+#else /* !CONFIG_SMP */
+static inline void pgtable_free_tlb(struct mmu_gather *tlb,
+                                    void *table, int shift)
+{
+        pgtable_free(table, shift);
+}
+#endif /* CONFIG_SMP */
+
+static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
+                                  unsigned long address)
+{
+        struct page *page = page_address(table);
+
+        tlb_flush_pgtable(tlb, address);
+        pgtable_page_dtor(page);
+        pgtable_free_tlb(tlb, page, 0);
+}
+
+#endif /* CONFIG_PPC_64K_PAGES */
+
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+        return kmem_cache_alloc(PGT_CACHE(PMD_INDEX_SIZE),
+                                GFP_KERNEL|__GFP_REPEAT);
+}
+
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+        kmem_cache_free(PGT_CACHE(PMD_INDEX_SIZE), pmd);
+}
+
+
 #define __pmd_free_tlb(tlb, pmd, addr)                \
         pgtable_free_tlb(tlb, pmd, PMD_INDEX_SIZE)
 #ifndef CONFIG_PPC_64K_PAGES
diff --git a/arch/powerpc/include/asm/pgalloc.h b/arch/powerpc/include/asm/pgalloc.h
index bf301ac62f35..e9a9f60e596d 100644
--- a/arch/powerpc/include/asm/pgalloc.h
+++ b/arch/powerpc/include/asm/pgalloc.h
@@ -3,6 +3,7 @@
 #ifdef __KERNEL__
 
 #include <linux/mm.h>
+#include <asm-generic/tlb.h>
 
 #ifdef CONFIG_PPC_BOOK3E
 extern void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address);
@@ -13,56 +14,11 @@ static inline void tlb_flush_pgtable(struct mmu_gather *tlb,
 }
 #endif /* !CONFIG_PPC_BOOK3E */
 
-static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
-{
-        free_page((unsigned long)pte);
-}
-
-static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
-{
-        pgtable_page_dtor(ptepage);
-        __free_page(ptepage);
-}
-
 #ifdef CONFIG_PPC64
 #include <asm/pgalloc-64.h>
 #else
 #include <asm/pgalloc-32.h>
 #endif
 
-#ifdef CONFIG_SMP
-struct mmu_gather;
-extern void tlb_remove_table(struct mmu_gather *, void *);
-
-static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
-{
-        unsigned long pgf = (unsigned long)table;
-        BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-        pgf |= shift;
-        tlb_remove_table(tlb, (void *)pgf);
-}
-
-static inline void __tlb_remove_table(void *_table)
-{
-        void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
-        unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
-
-        pgtable_free(table, shift);
-}
-#else /* CONFIG_SMP */
-static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, unsigned shift)
-{
-        pgtable_free(table, shift);
-}
-#endif /* !CONFIG_SMP */
-
-static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage,
-                                  unsigned long address)
-{
-        tlb_flush_pgtable(tlb, address);
-        pgtable_page_dtor(ptepage);
-        pgtable_free_tlb(tlb, page_address(ptepage), 0);
-}
-
 #endif /* __KERNEL__ */
 #endif /* _ASM_POWERPC_PGALLOC_H */