[IA64] Percpu quicklist for combined allocator for pgd/pmd/pte.

This patch introduces using the quicklists for pgd, pmd, and pte levels
by combining the alloc and free functions into a common set of routines.
This greatly simplifies the reading of this header file.

This patch is simple but necessary for large numa configurations.
It simply ensures that only pages from the local node are added to a
cpus quicklist.  This prevents the trapping of pages on a remote nodes
quicklist by starting a process, touching a large number of pages to
fill pmd and pte entries, migrating to another node, and then unmapping
or exiting.  With those conditions, the pages get trapped and if the
machine has more than 100 nodes of the same size, the calculation of
the pgtable high water mark will be larger than any single node so page
table cache flushing will never occur.

I ran lmbench lat_proc fork and lat_proc exec on a zx1 with and without
this patch and did not notice any change.

On an sn2 machine, there was a slight improvement which is possibly
due to pages from other nodes trapped on the test node before starting
the run.  I did not investigate further.

This patch shrinks the quicklist based upon free memory on the node
instead of the high/low water marks.  I have written it to enable
preemption periodically and recalculate the amount to shrink every time
we have freed enough pages that the quicklist size should have grown.
I rescan the nodes zones each pass because other processess may be
draining node memory at the same time as we are adding.

Signed-off-by: Robin Holt <holt@sgi.com>
Signed-off-by: Tony Luck <tony.luck@intel.com>

2 files changed, 61 insertions, 86 deletions

diff --git a/include/asm-ia64/pgalloc.h b/include/asm-ia64/pgalloc.h
index 0f05dc8bd46..e86a8c331ee 100644
--- a/include/asm-ia64/pgalloc.h
+++ b/include/asm-ia64/pgalloc.h
@@ -22,146 +22,124 @@
 
 #include <asm/mmu_context.h>
 
-/*
- * Very stupidly, we used to get new pgd's and pmd's, init their contents
- * to point to the NULL versions of the next level page table, later on
- * completely re-init them the same way, then free them up.  This wasted
- * a lot of work and caused unnecessary memory traffic.  How broken...
- * We fix this by caching them.
- */
-#define pgd_quicklist           (local_cpu_data->pgd_quick)
-#define pmd_quicklist           (local_cpu_data->pmd_quick)
-#define pgtable_cache_size      (local_cpu_data->pgtable_cache_sz)
+DECLARE_PER_CPU(unsigned long *, __pgtable_quicklist);
+#define pgtable_quicklist __ia64_per_cpu_var(__pgtable_quicklist)
+DECLARE_PER_CPU(long, __pgtable_quicklist_size);
+#define pgtable_quicklist_size __ia64_per_cpu_var(__pgtable_quicklist_size)
 
-static inline pgd_t*
-pgd_alloc_one_fast (struct mm_struct *mm)
+static inline long pgtable_quicklist_total_size(void)
+{
+        long ql_size;
+        int cpuid;
+
+        for_each_online_cpu(cpuid) {
+                ql_size += per_cpu(__pgtable_quicklist_size, cpuid);
+        }
+        return ql_size;
+}
+
+static inline void *pgtable_quicklist_alloc(void)
 {
         unsigned long *ret = NULL;
 
         preempt_disable();
 
-        ret = pgd_quicklist;
+        ret = pgtable_quicklist;
         if (likely(ret != NULL)) {
-                pgd_quicklist = (unsigned long *)(*ret);
+                pgtable_quicklist = (unsigned long *)(*ret);
                 ret[0] = 0;
-                --pgtable_cache_size;
-        } else
-                ret = NULL;
+                --pgtable_quicklist_size;
+        } else {
+                ret = (unsigned long *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
+        }
 
         preempt_enable();
 
-        return (pgd_t *) ret;
+        return ret;
 }
 
-static inline pgd_t*
-pgd_alloc (struct mm_struct *mm)
+static inline void pgtable_quicklist_free(void *pgtable_entry)
 {
-        /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */
-        pgd_t *pgd = pgd_alloc_one_fast(mm);
+#ifdef CONFIG_NUMA
+        unsigned long nid = page_to_nid(virt_to_page(pgtable_entry));
 
-        if (unlikely(pgd == NULL)) {
-                pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
+        if (unlikely(nid != numa_node_id())) {
+                free_page((unsigned long)pgtable_entry);
+                return;
         }
-        return pgd;
-}
+#endif
 
-static inline void
-pgd_free (pgd_t *pgd)
-{
         preempt_disable();
-        *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
-        pgd_quicklist = (unsigned long *) pgd;
-        ++pgtable_cache_size;
+        *(unsigned long *)pgtable_entry = (unsigned long)pgtable_quicklist;
+        pgtable_quicklist = (unsigned long *)pgtable_entry;
+        ++pgtable_quicklist_size;
         preempt_enable();
 }
 
-static inline void
-pud_populate (struct mm_struct *mm, pud_t *pud_entry, pmd_t *pmd)
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-        pud_val(*pud_entry) = __pa(pmd);
+        return pgtable_quicklist_alloc();
 }
 
-static inline pmd_t*
-pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr)
+static inline void pgd_free(pgd_t * pgd)
 {
-        unsigned long *ret = NULL;
-
-        preempt_disable();
-
-        ret = (unsigned long *)pmd_quicklist;
-        if (likely(ret != NULL)) {
-                pmd_quicklist = (unsigned long *)(*ret);
-                ret[0] = 0;
-                --pgtable_cache_size;
-        }
-
-        preempt_enable();
-
-        return (pmd_t *)ret;
+        pgtable_quicklist_free(pgd);
 }
 
-static inline pmd_t*
-pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
+static inline void
+pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
 {
-        pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
+        pud_val(*pud_entry) = __pa(pmd);
+}
 
-        return pmd;
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+        return pgtable_quicklist_alloc();
 }
 
-static inline void
-pmd_free (pmd_t *pmd)
+static inline void pmd_free(pmd_t * pmd)
 {
-        preempt_disable();
-        *(unsigned long *)pmd = (unsigned long) pmd_quicklist;
-        pmd_quicklist = (unsigned long *) pmd;
-        ++pgtable_cache_size;
-        preempt_enable();
+        pgtable_quicklist_free(pmd);
 }
 
 #define __pmd_free_tlb(tlb, pmd)        pmd_free(pmd)
 
 static inline void
-pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte)
+pmd_populate(struct mm_struct *mm, pmd_t * pmd_entry, struct page *pte)
 {
         pmd_val(*pmd_entry) = page_to_phys(pte);
 }
 
 static inline void
-pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte)
+pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte)
 {
         pmd_val(*pmd_entry) = __pa(pte);
 }
 
-static inline struct page *
-pte_alloc_one (struct mm_struct *mm, unsigned long addr)
+static inline struct page *pte_alloc_one(struct mm_struct *mm,
+                                         unsigned long addr)
 {
-        struct page *pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
-
-        return pte;
+        return virt_to_page(pgtable_quicklist_alloc());
 }
 
-static inline pte_t *
-pte_alloc_one_kernel (struct mm_struct *mm, unsigned long addr)
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+                                          unsigned long addr)
 {
-        pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
-
-        return pte;
+        return pgtable_quicklist_alloc();
 }
 
-static inline void
-pte_free (struct page *pte)
+static inline void pte_free(struct page *pte)
 {
-        __free_page(pte);
+        pgtable_quicklist_free(page_address(pte));
 }
 
-static inline void
-pte_free_kernel (pte_t *pte)
+static inline void pte_free_kernel(pte_t * pte)
 {
-        free_page((unsigned long) pte);
+        pgtable_quicklist_free(pte);
 }
 
-#define __pte_free_tlb(tlb, pte)        tlb_remove_page((tlb), (pte))
+#define __pte_free_tlb(tlb, pte)        pte_free(pte)
 
-extern void check_pgt_cache (void);
+extern void check_pgt_cache(void);
 
-#endif /* _ASM_IA64_PGALLOC_H */
+#endif                          /* _ASM_IA64_PGALLOC_H */
diff --git a/include/asm-ia64/processor.h b/include/asm-ia64/processor.h
index 2807f8d766d..983798ec179 100644
--- a/include/asm-ia64/processor.h
+++ b/include/asm-ia64/processor.h
@@ -137,9 +137,6 @@ struct cpuinfo_ia64 {
         __u64 nsec_per_cyc;     /* (1000000000<<IA64_NSEC_PER_CYC_SHIFT)/itc_freq */
         __u64 unimpl_va_mask;   /* mask of unimplemented virtual address bits (from PAL) */
         __u64 unimpl_pa_mask;   /* mask of unimplemented physical address bits (from PAL) */
-        __u64 *pgd_quick;
-        __u64 *pmd_quick;
-        __u64 pgtable_cache_sz;
         __u64 itc_freq;         /* frequency of ITC counter */
         __u64 proc_freq;        /* frequency of processor */
         __u64 cyc_per_usec;     /* itc_freq/1000000 */