aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-x86_64
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@woody.linux-foundation.org>2007-09-21 15:09:41 -0400
committerLinus Torvalds <torvalds@woody.linux-foundation.org>2007-09-21 15:09:41 -0400
commitda8f153e51290e7438ba7da66234a864e5d3e1c1 (patch)
tree90d3cc5eeb5d2e18877ccb1123a4df7873fd3c58 /include/asm-x86_64
parent077a6c26696c63305eebafbb71890c95e2750b6d (diff)
Revert "x86_64: Quicklist support for x86_64"
This reverts commit 34feb2c83beb3bdf13535a36770f7e50b47ef299. Suresh Siddha points out that this one breaks the fundamental requirement that you cannot free page table pages before the TLB caches are flushed. The quicklists do not give the same kinds of guarantees that the mmu_gather structure does, at least not in NUMA configurations. Requested-by: Suresh Siddha <suresh.b.siddha@intel.com> Acked-by: Andi Kleen <ak@suse.de> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Christoph Lameter <clameter@sgi.com> Cc: Asit Mallick <asit.k.mallick@intel.com> Cc: Tony Luck <tony.luck@intel.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include/asm-x86_64')
-rw-r--r--include/asm-x86_64/pgalloc.h73
-rw-r--r--include/asm-x86_64/pgtable.h1
2 files changed, 25 insertions, 49 deletions
diff --git a/include/asm-x86_64/pgalloc.h b/include/asm-x86_64/pgalloc.h
index b467be6d367f..8bb564687860 100644
--- a/include/asm-x86_64/pgalloc.h
+++ b/include/asm-x86_64/pgalloc.h
@@ -4,10 +4,6 @@
4#include <asm/pda.h> 4#include <asm/pda.h>
5#include <linux/threads.h> 5#include <linux/threads.h>
6#include <linux/mm.h> 6#include <linux/mm.h>
7#include <linux/quicklist.h>
8
9#define QUICK_PGD 0 /* We preserve special mappings over free */
10#define QUICK_PT 1 /* Other page table pages that are zero on free */
11 7
12#define pmd_populate_kernel(mm, pmd, pte) \ 8#define pmd_populate_kernel(mm, pmd, pte) \
13 set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte))) 9 set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte)))
@@ -24,23 +20,23 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *p
24static inline void pmd_free(pmd_t *pmd) 20static inline void pmd_free(pmd_t *pmd)
25{ 21{
26 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); 22 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
27 quicklist_free(QUICK_PT, NULL, pmd); 23 free_page((unsigned long)pmd);
28} 24}
29 25
30static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr) 26static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
31{ 27{
32 return (pmd_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 28 return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
33} 29}
34 30
35static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 31static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
36{ 32{
37 return (pud_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 33 return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
38} 34}
39 35
40static inline void pud_free (pud_t *pud) 36static inline void pud_free (pud_t *pud)
41{ 37{
42 BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); 38 BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
43 quicklist_free(QUICK_PT, NULL, pud); 39 free_page((unsigned long)pud);
44} 40}
45 41
46static inline void pgd_list_add(pgd_t *pgd) 42static inline void pgd_list_add(pgd_t *pgd)
@@ -61,57 +57,41 @@ static inline void pgd_list_del(pgd_t *pgd)
61 spin_unlock(&pgd_lock); 57 spin_unlock(&pgd_lock);
62} 58}
63 59
64static inline void pgd_ctor(void *x) 60static inline pgd_t *pgd_alloc(struct mm_struct *mm)
65{ 61{
66 unsigned boundary; 62 unsigned boundary;
67 pgd_t *pgd = x; 63 pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT);
68 struct page *page = virt_to_page(pgd); 64 if (!pgd)
69 65 return NULL;
66 pgd_list_add(pgd);
70 /* 67 /*
71 * Copy kernel pointers in from init. 68 * Copy kernel pointers in from init.
69 * Could keep a freelist or slab cache of those because the kernel
70 * part never changes.
72 */ 71 */
73 boundary = pgd_index(__PAGE_OFFSET); 72 boundary = pgd_index(__PAGE_OFFSET);
73 memset(pgd, 0, boundary * sizeof(pgd_t));
74 memcpy(pgd + boundary, 74 memcpy(pgd + boundary,
75 init_level4_pgt + boundary, 75 init_level4_pgt + boundary,
76 (PTRS_PER_PGD - boundary) * sizeof(pgd_t)); 76 (PTRS_PER_PGD - boundary) * sizeof(pgd_t));
77
78 spin_lock(&pgd_lock);
79 list_add(&page->lru, &pgd_list);
80 spin_unlock(&pgd_lock);
81}
82
83static inline void pgd_dtor(void *x)
84{
85 pgd_t *pgd = x;
86 struct page *page = virt_to_page(pgd);
87
88 spin_lock(&pgd_lock);
89 list_del(&page->lru);
90 spin_unlock(&pgd_lock);
91}
92
93static inline pgd_t *pgd_alloc(struct mm_struct *mm)
94{
95 pgd_t *pgd = (pgd_t *)quicklist_alloc(QUICK_PGD,
96 GFP_KERNEL|__GFP_REPEAT, pgd_ctor);
97 return pgd; 77 return pgd;
98} 78}
99 79
100static inline void pgd_free(pgd_t *pgd) 80static inline void pgd_free(pgd_t *pgd)
101{ 81{
102 BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); 82 BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
103 quicklist_free(QUICK_PGD, pgd_dtor, pgd); 83 pgd_list_del(pgd);
84 free_page((unsigned long)pgd);
104} 85}
105 86
106static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) 87static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
107{ 88{
108 return (pte_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 89 return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
109} 90}
110 91
111static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) 92static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
112{ 93{
113 void *p = (void *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); 94 void *p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT);
114
115 if (!p) 95 if (!p)
116 return NULL; 96 return NULL;
117 return virt_to_page(p); 97 return virt_to_page(p);
@@ -123,22 +103,17 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long add
123static inline void pte_free_kernel(pte_t *pte) 103static inline void pte_free_kernel(pte_t *pte)
124{ 104{
125 BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); 105 BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
126 quicklist_free(QUICK_PT, NULL, pte); 106 free_page((unsigned long)pte);
127} 107}
128 108
129static inline void pte_free(struct page *pte) 109static inline void pte_free(struct page *pte)
130{ 110{
131 quicklist_free_page(QUICK_PT, NULL, pte); 111 __free_page(pte);
132} 112}
133 113
134#define __pte_free_tlb(tlb,pte) quicklist_free_page(QUICK_PT, NULL,(pte)) 114#define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
135 115
136#define __pmd_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x)) 116#define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
137#define __pud_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x)) 117#define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x))
138 118
139static inline void check_pgt_cache(void)
140{
141 quicklist_trim(QUICK_PGD, pgd_dtor, 25, 16);
142 quicklist_trim(QUICK_PT, NULL, 25, 16);
143}
144#endif /* _X86_64_PGALLOC_H */ 119#endif /* _X86_64_PGALLOC_H */
diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h
index c9d8764c89d1..57dd6b3107ea 100644
--- a/include/asm-x86_64/pgtable.h
+++ b/include/asm-x86_64/pgtable.h
@@ -411,6 +411,7 @@ pte_t *lookup_address(unsigned long addr);
411#define HAVE_ARCH_UNMAPPED_AREA 411#define HAVE_ARCH_UNMAPPED_AREA
412 412
413#define pgtable_cache_init() do { } while (0) 413#define pgtable_cache_init() do { } while (0)
414#define check_pgt_cache() do { } while (0)
414 415
415#define PAGE_AGP PAGE_KERNEL_NOCACHE 416#define PAGE_AGP PAGE_KERNEL_NOCACHE
416#define HAVE_PAGE_AGP 1 417#define HAVE_PAGE_AGP 1