1 files changed, 61 insertions, 83 deletions

diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h
index 0f05dc8bd46..e86a8c331ee 100644
--- a/include/asm-ia64/pgalloc.h
+++ b/include/asm-ia64/pgalloc.h
@@ -22,146 +22,124 @@
 
 #include <asm/mmu_context.h>
 
-/*
- * Very stupidly, we used to get new pgd's and pmd's, init their contents
- * to point to the NULL versions of the next level page table, later on
- * completely re-init them the same way, then free them up.  This wasted
- * a lot of work and caused unnecessary memory traffic.  How broken...
- * We fix this by caching them.
- */
-#define pgd_quicklist           (local_cpu_data->pgd_quick)
-#define pmd_quicklist           (local_cpu_data->pmd_quick)
-#define pgtable_cache_size      (local_cpu_data->pgtable_cache_sz)
+DECLARE_PER_CPU(unsigned long *, __pgtable_quicklist);
+#define pgtable_quicklist __ia64_per_cpu_var(__pgtable_quicklist)
+DECLARE_PER_CPU(long, __pgtable_quicklist_size);
+#define pgtable_quicklist_size __ia64_per_cpu_var(__pgtable_quicklist_size)
 
-static inline pgd_t*
-pgd_alloc_one_fast (struct mm_struct *mm)
+static inline long pgtable_quicklist_total_size(void)
+{
+        long ql_size;
+        int cpuid;
+
+        for_each_online_cpu(cpuid) {
+                ql_size += per_cpu(__pgtable_quicklist_size, cpuid);
+        }
+        return ql_size;
+}
+
+static inline void *pgtable_quicklist_alloc(void)
 {
         unsigned long *ret = NULL;
 
         preempt_disable();
 
-        ret = pgd_quicklist;
+        ret = pgtable_quicklist;
         if (likely(ret != NULL)) {
-                pgd_quicklist = (unsigned long *)(*ret);
+                pgtable_quicklist = (unsigned long *)(*ret);
                 ret[0] = 0;
-                --pgtable_cache_size;
-        } else
-                ret = NULL;
+                --pgtable_quicklist_size;
+        } else {
+                ret = (unsigned long *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+        }
 
         preempt_enable();
 
-        return (pgd_t *) ret;
+        return ret;
 }
 
-static inline pgd_t*
-pgd_alloc (struct mm_struct *mm)
+static inline void pgtable_quicklist_free(void *pgtable_entry)
 {
-        /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */
-        pgd_t *pgd = pgd_alloc_one_fast(mm);
+#ifdef CONFIG_NUMA
+        unsigned long nid = page_to_nid(virt_to_page(pgtable_entry));
 
-        if (unlikely(pgd == NULL)) {
-                pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
+        if (unlikely(nid != numa_node_id())) {
+                free_page((unsigned long)pgtable_entry);
+                return;
         }
-        return pgd;
-}
+#endif
 
-static inline void
-pgd_free (pgd_t *pgd)
-{
         preempt_disable();
-        *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
-        pgd_quicklist = (unsigned long *) pgd;
-        ++pgtable_cache_size;
+        *(unsigned long *)pgtable_entry = (unsigned long)pgtable_quicklist;
+        pgtable_quicklist = (unsigned long *)pgtable_entry;
+        ++pgtable_quicklist_size;
         preempt_enable();
 }
 
-static inline void
-pud_populate (struct mm_struct *mm, pud_t *pud_entry, pmd_t *pmd)
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-        pud_val(*pud_entry) = __pa(pmd);
+        return pgtable_quicklist_alloc();
 }
 
-static inline pmd_t*
-pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr)
+static inline void pgd_free(pgd_t * pgd)
 {
-        unsigned long *ret = NULL;
-
-        preempt_disable();
-
-        ret = (unsigned long *)pmd_quicklist;
-        if (likely(ret != NULL)) {
-                pmd_quicklist = (unsigned long *)(*ret);
-                ret[0] = 0;
-                --pgtable_cache_size;
-        }
-
-        preempt_enable();
-
-        return (pmd_t *)ret;
+        pgtable_quicklist_free(pgd);
 }
 
-static inline pmd_t*
-pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
+static inline void
+pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
 {
-        pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
+        pud_val(*pud_entry) = __pa(pmd);
+}
 
-        return pmd;
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+        return pgtable_quicklist_alloc();
 }
 
-static inline void
-pmd_free (pmd_t *pmd)
+static inline void pmd_free(pmd_t * pmd)
 {
-        preempt_disable();
-        *(unsigned long *)pmd = (unsigned long) pmd_quicklist;
-        pmd_quicklist = (unsigned long *) pmd;
-        ++pgtable_cache_size;
-        preempt_enable();
+        pgtable_quicklist_free(pmd);
 }
 
 #define __pmd_free_tlb(tlb, pmd)        pmd_free(pmd)
 
 static inline void
-pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte)
+pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, struct page *pte)
 {
         pmd_val(*pmd_entry) = page_to_phys(pte);
 }
 
 static inline void
-pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte)
+pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte)
 {
         pmd_val(*pmd_entry) = __pa(pte);
 }
 
-static inline struct page *
-pte_alloc_one (struct mm_struct *mm, unsigned long addr)
+static inline struct page *pte_alloc_one(struct mm_struct *mm,
+                                         unsigned long addr)
 {
-        struct page *pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
-
-        return pte;
+        return virt_to_page(pgtable_quicklist_alloc());
 }
 
-static inline pte_t *
-pte_alloc_one_kernel (struct mm_struct *mm, unsigned long addr)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                                          unsigned long addr)
 {
-        pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
-
-        return pte;
+        return pgtable_quicklist_alloc();
 }
 
-static inline void
-pte_free (struct page *pte)
+static inline void pte_free(struct page *pte)
 {
-        __free_page(pte);
+        pgtable_quicklist_free(page_address(pte));
 }
 
-static inline void
-pte_free_kernel (pte_t *pte)
+static inline void pte_free_kernel(pte_t * pte)
 {
-        free_page((unsigned long) pte);
+        pgtable_quicklist_free(pte);
 }
 
-#define __pte_free_tlb(tlb, pte)        tlb_remove_page((tlb), (pte))
+#define __pte_free_tlb(tlb, pte)        pte_free(pte)
 
-extern void check_pgt_cache (void);
+extern void check_pgt_cache(void);
 
-#endif /* _ASM_IA64_PGALLOC_H */
+#endif                          /* _ASM_IA64_PGALLOC_H */