aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/x86_64/Kconfig8
-rw-r--r--arch/x86_64/kernel/process.c1
-rw-r--r--arch/x86_64/kernel/smp.c2
-rw-r--r--include/asm-x86_64/pgalloc.h73
-rw-r--r--include/asm-x86_64/pgtable.h1
5 files changed, 26 insertions, 59 deletions
diff --git a/arch/x86_64/Kconfig b/arch/x86_64/Kconfig
index ffa036406289..b4d9089a6a06 100644
--- a/arch/x86_64/Kconfig
+++ b/arch/x86_64/Kconfig
@@ -60,14 +60,6 @@ config ZONE_DMA
60 bool 60 bool
61 default y 61 default y
62 62
63config QUICKLIST
64 bool
65 default y
66
67config NR_QUICK
68 int
69 default 2
70
71config ISA 63config ISA
72 bool 64 bool
73 65
diff --git a/arch/x86_64/kernel/process.c b/arch/x86_64/kernel/process.c
index 2842f50cbe3f..98956555450b 100644
--- a/arch/x86_64/kernel/process.c
+++ b/arch/x86_64/kernel/process.c
@@ -208,7 +208,6 @@ void cpu_idle (void)
208 if (__get_cpu_var(cpu_idle_state)) 208 if (__get_cpu_var(cpu_idle_state))
209 __get_cpu_var(cpu_idle_state) = 0; 209 __get_cpu_var(cpu_idle_state) = 0;
210 210
211 check_pgt_cache();
212 rmb(); 211 rmb();
213 idle = pm_idle; 212 idle = pm_idle;
214 if (!idle) 213 if (!idle)
diff --git a/arch/x86_64/kernel/smp.c b/arch/x86_64/kernel/smp.c
index 673a300b5944..df4a82812adb 100644
--- a/arch/x86_64/kernel/smp.c
+++ b/arch/x86_64/kernel/smp.c
@@ -241,7 +241,7 @@ void flush_tlb_mm (struct mm_struct * mm)
241 } 241 }
242 if (!cpus_empty(cpu_mask)) 242 if (!cpus_empty(cpu_mask))
243 flush_tlb_others(cpu_mask, mm, FLUSH_ALL); 243 flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
244 check_pgt_cache(); 244
245 preempt_enable(); 245 preempt_enable();
246} 246}
247EXPORT_SYMBOL(flush_tlb_mm); 247EXPORT_SYMBOL(flush_tlb_mm);
diff --git a/include/asm-x86_64/pgalloc.h b/include/asm-x86_64/pgalloc.h
index b467be6d367f..8bb564687860 100644
--- a/include/asm-x86_64/pgalloc.h
+++ b/include/asm-x86_64/pgalloc.h
@@ -4,10 +4,6 @@
4#include <asm/pda.h> 4#include <asm/pda.h>
5#include <linux/threads.h> 5#include <linux/threads.h>
6#include <linux/mm.h> 6#include <linux/mm.h>
7#include <linux/quicklist.h>
8
9#define QUICK_PGD 0 /* We preserve special mappings over free */
10#define QUICK_PT 1 /* Other page table pages that are zero on free */
11 7
12#define pmd_populate_kernel(mm, pmd, pte) \ 8#define pmd_populate_kernel(mm, pmd, pte) \
13 set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte))) 9 set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)))
@@ -24,23 +20,23 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *p
24static inline void pmd_free(pmd_t *pmd) 20static inline void pmd_free(pmd_t *pmd)
25{ 21{
26 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); 22 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
27 quicklist_free(QUICK_PT, NULL, pmd); 23 free_page((unsigned long)pmd);
28} 24}
29 25
30static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr) 26static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
31{ 27{
32 return (pmd_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 28 return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
33} 29}
34 30
35static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 31static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
36{ 32{
37 return (pud_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 33 return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
38} 34}
39 35
40static inline void pud_free (pud_t *pud) 36static inline void pud_free (pud_t *pud)
41{ 37{
42 BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); 38 BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
43 quicklist_free(QUICK_PT, NULL, pud); 39 free_page((unsigned long)pud);
44} 40}
45 41
46static inline void pgd_list_add(pgd_t *pgd) 42static inline void pgd_list_add(pgd_t *pgd)
@@ -61,57 +57,41 @@ static inline void pgd_list_del(pgd_t *pgd)
61 spin_unlock(&pgd_lock); 57 spin_unlock(&pgd_lock);
62} 58}
63 59
64static inline void pgd_ctor(void *x) 60static inline pgd_t *pgd_alloc(struct mm_struct *mm)
65{ 61{
66 unsigned boundary; 62 unsigned boundary;
67 pgd_t *pgd = x; 63 pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT);
68 struct page *page = virt_to_page(pgd); 64 if (!pgd)
69 65 return NULL;
66 pgd_list_add(pgd);
70 /* 67 /*
71 * Copy kernel pointers in from init. 68 * Copy kernel pointers in from init.
69 * Could keep a freelist or slab cache of those because the kernel
70 * part never changes.
72 */ 71 */
73 boundary = pgd_index(__PAGE_OFFSET); 72 boundary = pgd_index(__PAGE_OFFSET);
73 memset(pgd, 0, boundary * sizeof(pgd_t));
74 memcpy(pgd + boundary, 74 memcpy(pgd + boundary,
75 init_level4_pgt + boundary, 75 init_level4_pgt + boundary,
76 (PTRS_PER_PGD - boundary) * sizeof(pgd_t)); 76 (PTRS_PER_PGD - boundary) * sizeof(pgd_t));
77
78 spin_lock(&pgd_lock);
79 list_add(&page->lru, &pgd_list);
80 spin_unlock(&pgd_lock);
81}
82
83static inline void pgd_dtor(void *x)
84{
85 pgd_t *pgd = x;
86 struct page *page = virt_to_page(pgd);
87
88 spin_lock(&pgd_lock);
89 list_del(&page->lru);
90 spin_unlock(&pgd_lock);
91}
92
93static inline pgd_t *pgd_alloc(struct mm_struct *mm)
94{
95 pgd_t *pgd = (pgd_t *)quicklist_alloc(QUICK_PGD,
96 GFP_KERNEL|__GFP_REPEAT, pgd_ctor);
97 return pgd; 77 return pgd;
98} 78}
99 79
100static inline void pgd_free(pgd_t *pgd) 80static inline void pgd_free(pgd_t *pgd)
101{ 81{
102 BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); 82 BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
103 quicklist_free(QUICK_PGD, pgd_dtor, pgd); 83 pgd_list_del(pgd);
84 free_page((unsigned long)pgd);
104} 85}
105 86
106static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) 87static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
107{ 88{
108 return (pte_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 89 return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
109} 90}
110 91
111static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) 92static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
112{ 93{
113 void *p = (void *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 94 void *p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
114
115 if (!p) 95 if (!p)
116 return NULL; 96 return NULL;
117 return virt_to_page(p); 97 return virt_to_page(p);
@@ -123,22 +103,17 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long add
123static inline void pte_free_kernel(pte_t *pte) 103static inline void pte_free_kernel(pte_t *pte)
124{ 104{
125 BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); 105 BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
126 quicklist_free(QUICK_PT, NULL, pte); 106 free_page((unsigned long)pte);
127} 107}
128 108
129static inline void pte_free(struct page *pte) 109static inline void pte_free(struct page *pte)
130{ 110{
131 quicklist_free_page(QUICK_PT, NULL, pte); 111 __free_page(pte);
132} 112}
133 113
134#define __pte_free_tlb(tlb,pte) quicklist_free_page(QUICK_PT, NULL,(pte)) 114#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
135 115
136#define __pmd_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x)) 116#define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
137#define __pud_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x)) 117#define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
138 118
139static inline void check_pgt_cache(void)
140{
141 quicklist_trim(QUICK_PGD, pgd_dtor, 25, 16);
142 quicklist_trim(QUICK_PT, NULL, 25, 16);
143}
144#endif /* _X86_64_PGALLOC_H */ 119#endif /* _X86_64_PGALLOC_H */
diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h
index c9d8764c89d1..57dd6b3107ea 100644
--- a/include/asm-x86_64/pgtable.h
+++ b/include/asm-x86_64/pgtable.h
@@ -411,6 +411,7 @@ pte_t *lookup_address(unsigned long addr);
411#define HAVE_ARCH_UNMAPPED_AREA 411#define HAVE_ARCH_UNMAPPED_AREA
412 412
413#define pgtable_cache_init() do { } while (0) 413#define pgtable_cache_init() do { } while (0)
414#define check_pgt_cache() do { } while (0)
414 415
415#define PAGE_AGP PAGE_KERNEL_NOCACHE 416#define PAGE_AGP PAGE_KERNEL_NOCACHE
416#define HAVE_PAGE_AGP 1 417#define HAVE_PAGE_AGP 1