diff options
Diffstat (limited to 'include/asm-x86_64/pgalloc.h')
-rw-r--r-- | include/asm-x86_64/pgalloc.h | 73 |
1 files changed, 24 insertions, 49 deletions
diff --git a/include/asm-x86_64/pgalloc.h b/include/asm-x86_64/pgalloc.h index b467be6d367f..8bb564687860 100644 --- a/include/asm-x86_64/pgalloc.h +++ b/include/asm-x86_64/pgalloc.h | |||
@@ -4,10 +4,6 @@ | |||
4 | #include <asm/pda.h> | 4 | #include <asm/pda.h> |
5 | #include <linux/threads.h> | 5 | #include <linux/threads.h> |
6 | #include <linux/mm.h> | 6 | #include <linux/mm.h> |
7 | #include <linux/quicklist.h> | ||
8 | |||
9 | #define QUICK_PGD 0 /* We preserve special mappings over free */ | ||
10 | #define QUICK_PT 1 /* Other page table pages that are zero on free */ | ||
11 | 7 | ||
12 | #define pmd_populate_kernel(mm, pmd, pte) \ | 8 | #define pmd_populate_kernel(mm, pmd, pte) \ |
13 | set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte))) | 9 | set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte))) |
@@ -24,23 +20,23 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, struct page *p | |||
24 | static inline void pmd_free(pmd_t *pmd) | 20 | static inline void pmd_free(pmd_t *pmd) |
25 | { | 21 | { |
26 | BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); | 22 | BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); |
27 | quicklist_free(QUICK_PT, NULL, pmd); | 23 | free_page((unsigned long)pmd); |
28 | } | 24 | } |
29 | 25 | ||
30 | static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr) | 26 | static inline pmd_t *pmd_alloc_one (struct mm_struct *mm, unsigned long addr) |
31 | { | 27 | { |
32 | return (pmd_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); | 28 | return (pmd_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); |
33 | } | 29 | } |
34 | 30 | ||
35 | static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) | 31 | static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) |
36 | { | 32 | { |
37 | return (pud_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); | 33 | return (pud_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); |
38 | } | 34 | } |
39 | 35 | ||
40 | static inline void pud_free (pud_t *pud) | 36 | static inline void pud_free (pud_t *pud) |
41 | { | 37 | { |
42 | BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); | 38 | BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); |
43 | quicklist_free(QUICK_PT, NULL, pud); | 39 | free_page((unsigned long)pud); |
44 | } | 40 | } |
45 | 41 | ||
46 | static inline void pgd_list_add(pgd_t *pgd) | 42 | static inline void pgd_list_add(pgd_t *pgd) |
@@ -61,57 +57,41 @@ static inline void pgd_list_del(pgd_t *pgd) | |||
61 | spin_unlock(&pgd_lock); | 57 | spin_unlock(&pgd_lock); |
62 | } | 58 | } |
63 | 59 | ||
64 | static inline void pgd_ctor(void *x) | 60 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) |
65 | { | 61 | { |
66 | unsigned boundary; | 62 | unsigned boundary; |
67 | pgd_t *pgd = x; | 63 | pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT); |
68 | struct page *page = virt_to_page(pgd); | 64 | if (!pgd) |
69 | 65 | return NULL; | |
66 | pgd_list_add(pgd); | ||
70 | /* | 67 | /* |
71 | * Copy kernel pointers in from init. | 68 | * Copy kernel pointers in from init. |
69 | * Could keep a freelist or slab cache of those because the kernel | ||
70 | * part never changes. | ||
72 | */ | 71 | */ |
73 | boundary = pgd_index(__PAGE_OFFSET); | 72 | boundary = pgd_index(__PAGE_OFFSET); |
73 | memset(pgd, 0, boundary * sizeof(pgd_t)); | ||
74 | memcpy(pgd + boundary, | 74 | memcpy(pgd + boundary, |
75 | init_level4_pgt + boundary, | 75 | init_level4_pgt + boundary, |
76 | (PTRS_PER_PGD - boundary) * sizeof(pgd_t)); | 76 | (PTRS_PER_PGD - boundary) * sizeof(pgd_t)); |
77 | |||
78 | spin_lock(&pgd_lock); | ||
79 | list_add(&page->lru, &pgd_list); | ||
80 | spin_unlock(&pgd_lock); | ||
81 | } | ||
82 | |||
83 | static inline void pgd_dtor(void *x) | ||
84 | { | ||
85 | pgd_t *pgd = x; | ||
86 | struct page *page = virt_to_page(pgd); | ||
87 | |||
88 | spin_lock(&pgd_lock); | ||
89 | list_del(&page->lru); | ||
90 | spin_unlock(&pgd_lock); | ||
91 | } | ||
92 | |||
93 | static inline pgd_t *pgd_alloc(struct mm_struct *mm) | ||
94 | { | ||
95 | pgd_t *pgd = (pgd_t *)quicklist_alloc(QUICK_PGD, | ||
96 | GFP_KERNEL|__GFP_REPEAT, pgd_ctor); | ||
97 | return pgd; | 77 | return pgd; |
98 | } | 78 | } |
99 | 79 | ||
100 | static inline void pgd_free(pgd_t *pgd) | 80 | static inline void pgd_free(pgd_t *pgd) |
101 | { | 81 | { |
102 | BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); | 82 | BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); |
103 | quicklist_free(QUICK_PGD, pgd_dtor, pgd); | 83 | pgd_list_del(pgd); |
84 | free_page((unsigned long)pgd); | ||
104 | } | 85 | } |
105 | 86 | ||
106 | static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) | 87 | static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) |
107 | { | 88 | { |
108 | return (pte_t *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); | 89 | return (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); |
109 | } | 90 | } |
110 | 91 | ||
111 | static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) | 92 | static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) |
112 | { | 93 | { |
113 | void *p = (void *)quicklist_alloc(QUICK_PT, GFP_KERNEL|__GFP_REPEAT, NULL); | 94 | void *p = (void *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); |
114 | |||
115 | if (!p) | 95 | if (!p) |
116 | return NULL; | 96 | return NULL; |
117 | return virt_to_page(p); | 97 | return virt_to_page(p); |
@@ -123,22 +103,17 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm, unsigned long add | |||
123 | static inline void pte_free_kernel(pte_t *pte) | 103 | static inline void pte_free_kernel(pte_t *pte) |
124 | { | 104 | { |
125 | BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); | 105 | BUG_ON((unsigned long)pte & (PAGE_SIZE-1)); |
126 | quicklist_free(QUICK_PT, NULL, pte); | 106 | free_page((unsigned long)pte); |
127 | } | 107 | } |
128 | 108 | ||
129 | static inline void pte_free(struct page *pte) | 109 | static inline void pte_free(struct page *pte) |
130 | { | 110 | { |
131 | quicklist_free_page(QUICK_PT, NULL, pte); | 111 | __free_page(pte); |
132 | } | 112 | } |
133 | 113 | ||
134 | #define __pte_free_tlb(tlb,pte) quicklist_free_page(QUICK_PT, NULL,(pte)) | 114 | #define __pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte)) |
135 | 115 | ||
136 | #define __pmd_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x)) | 116 | #define __pmd_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) |
137 | #define __pud_free_tlb(tlb,x) quicklist_free(QUICK_PT, NULL, (x)) | 117 | #define __pud_free_tlb(tlb,x) tlb_remove_page((tlb),virt_to_page(x)) |
138 | 118 | ||
139 | static inline void check_pgt_cache(void) | ||
140 | { | ||
141 | quicklist_trim(QUICK_PGD, pgd_dtor, 25, 16); | ||
142 | quicklist_trim(QUICK_PT, NULL, 25, 16); | ||
143 | } | ||
144 | #endif /* _X86_64_PGALLOC_H */ | 119 | #endif /* _X86_64_PGALLOC_H */ |