diff options
author | Dean Nelson <dcn@sgi.com> | 2006-06-23 05:03:21 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@g5.osdl.org> | 2006-06-23 10:42:49 -0400 |
commit | 929f97276bcf7f4a95272ed08a85339b98ba210d (patch) | |
tree | 4975698af9559279c83e4e268213ed13e3efee9a /arch/ia64/kernel/uncached.c | |
parent | 833423143c3a7c6545e409d65febd0d92deb351b (diff) |
[PATCH] change gen_pool allocator to not touch managed memory
Modify the gen_pool allocator (lib/genalloc.c) to utilize a bitmap scheme
instead of the buddy scheme. The purpose of this change is to eliminate
the touching of the actual memory being allocated.
Since the change modifies the interface, a change to the uncached allocator
(arch/ia64/kernel/uncached.c) is also required.
Both Andrey Volkov and Jes Sorenson have expressed a desire that the
gen_pool allocator not write to the memory being managed. See the
following:
http://marc.theaimsgroup.com/?l=linux-kernel&m=113518602713125&w=2
http://marc.theaimsgroup.com/?l=linux-kernel&m=113533568827916&w=2
Signed-off-by: Dean Nelson <dcn@sgi.com>
Cc: Andrey Volkov <avolkov@varma-el.com>
Acked-by: Jes Sorensen <jes@trained-monkey.org>
Cc: "Luck, Tony" <tony.luck@intel.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'arch/ia64/kernel/uncached.c')
-rw-r--r-- | arch/ia64/kernel/uncached.c | 200 |
1 files changed, 105 insertions, 95 deletions
diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c index fcd2bad0286f..5f03b9e524dd 100644 --- a/arch/ia64/kernel/uncached.c +++ b/arch/ia64/kernel/uncached.c | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved. | 2 | * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms of version 2 of the GNU General Public License | 5 | * under the terms of version 2 of the GNU General Public License |
@@ -29,15 +29,8 @@ | |||
29 | #include <asm/tlbflush.h> | 29 | #include <asm/tlbflush.h> |
30 | #include <asm/sn/arch.h> | 30 | #include <asm/sn/arch.h> |
31 | 31 | ||
32 | #define DEBUG 0 | ||
33 | 32 | ||
34 | #if DEBUG | 33 | extern void __init efi_memmap_walk_uc(efi_freemem_callback_t, void *); |
35 | #define dprintk printk | ||
36 | #else | ||
37 | #define dprintk(x...) do { } while (0) | ||
38 | #endif | ||
39 | |||
40 | void __init efi_memmap_walk_uc (efi_freemem_callback_t callback); | ||
41 | 34 | ||
42 | #define MAX_UNCACHED_GRANULES 5 | 35 | #define MAX_UNCACHED_GRANULES 5 |
43 | static int allocated_granules; | 36 | static int allocated_granules; |
@@ -60,6 +53,7 @@ static void uncached_ipi_visibility(void *data) | |||
60 | static void uncached_ipi_mc_drain(void *data) | 53 | static void uncached_ipi_mc_drain(void *data) |
61 | { | 54 | { |
62 | int status; | 55 | int status; |
56 | |||
63 | status = ia64_pal_mc_drain(); | 57 | status = ia64_pal_mc_drain(); |
64 | if (status) | 58 | if (status) |
65 | printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on " | 59 | printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on " |
@@ -67,30 +61,35 @@ static void uncached_ipi_mc_drain(void *data) | |||
67 | } | 61 | } |
68 | 62 | ||
69 | 63 | ||
70 | static unsigned long | 64 | /* |
71 | uncached_get_new_chunk(struct gen_pool *poolp) | 65 | * Add a new chunk of uncached memory pages to the specified pool. |
66 | * | ||
67 | * @pool: pool to add new chunk of uncached memory to | ||
68 | * @nid: node id of node to allocate memory from, or -1 | ||
69 | * | ||
70 | * This is accomplished by first allocating a granule of cached memory pages | ||
71 | * and then converting them to uncached memory pages. | ||
72 | */ | ||
73 | static int uncached_add_chunk(struct gen_pool *pool, int nid) | ||
72 | { | 74 | { |
73 | struct page *page; | 75 | struct page *page; |
74 | void *tmp; | ||
75 | int status, i; | 76 | int status, i; |
76 | unsigned long addr, node; | 77 | unsigned long c_addr, uc_addr; |
77 | 78 | ||
78 | if (allocated_granules >= MAX_UNCACHED_GRANULES) | 79 | if (allocated_granules >= MAX_UNCACHED_GRANULES) |
79 | return 0; | 80 | return -1; |
81 | |||
82 | /* attempt to allocate a granule's worth of cached memory pages */ | ||
80 | 83 | ||
81 | node = poolp->private; | 84 | page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO, |
82 | page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, | ||
83 | IA64_GRANULE_SHIFT-PAGE_SHIFT); | 85 | IA64_GRANULE_SHIFT-PAGE_SHIFT); |
86 | if (!page) | ||
87 | return -1; | ||
84 | 88 | ||
85 | dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n", | 89 | /* convert the memory pages from cached to uncached */ |
86 | page, (unsigned long)(page-vmem_map) << PAGE_SHIFT); | ||
87 | 90 | ||
88 | /* | 91 | c_addr = (unsigned long)page_address(page); |
89 | * Do magic if no mem on local node! XXX | 92 | uc_addr = c_addr - PAGE_OFFSET + __IA64_UNCACHED_OFFSET; |
90 | */ | ||
91 | if (!page) | ||
92 | return 0; | ||
93 | tmp = page_address(page); | ||
94 | 93 | ||
95 | /* | 94 | /* |
96 | * There's a small race here where it's possible for someone to | 95 | * There's a small race here where it's possible for someone to |
@@ -100,76 +99,90 @@ uncached_get_new_chunk(struct gen_pool *poolp) | |||
100 | for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++) | 99 | for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++) |
101 | SetPageUncached(&page[i]); | 100 | SetPageUncached(&page[i]); |
102 | 101 | ||
103 | flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE); | 102 | flush_tlb_kernel_range(uc_addr, uc_adddr + IA64_GRANULE_SIZE); |
104 | 103 | ||
105 | status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); | 104 | status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); |
106 | |||
107 | dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n", | ||
108 | status, raw_smp_processor_id()); | ||
109 | |||
110 | if (!status) { | 105 | if (!status) { |
111 | status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1); | 106 | status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1); |
112 | if (status) | 107 | if (status) |
113 | printk(KERN_WARNING "smp_call_function failed for " | 108 | goto failed; |
114 | "uncached_ipi_visibility! (%i)\n", status); | ||
115 | } | 109 | } |
116 | 110 | ||
111 | preempt_disable(); | ||
112 | |||
117 | if (ia64_platform_is("sn2")) | 113 | if (ia64_platform_is("sn2")) |
118 | sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE); | 114 | sn_flush_all_caches(uc_addr, IA64_GRANULE_SIZE); |
119 | else | 115 | else |
120 | flush_icache_range((unsigned long)tmp, | 116 | flush_icache_range(uc_addr, uc_addr + IA64_GRANULE_SIZE); |
121 | (unsigned long)tmp+IA64_GRANULE_SIZE); | 117 | |
118 | /* flush the just introduced uncached translation from the TLB */ | ||
119 | local_flush_tlb_all(); | ||
120 | |||
121 | preempt_enable(); | ||
122 | 122 | ||
123 | ia64_pal_mc_drain(); | 123 | ia64_pal_mc_drain(); |
124 | status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1); | 124 | status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1); |
125 | if (status) | 125 | if (status) |
126 | printk(KERN_WARNING "smp_call_function failed for " | 126 | goto failed; |
127 | "uncached_ipi_mc_drain! (%i)\n", status); | ||
128 | 127 | ||
129 | addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET; | 128 | /* |
129 | * The chunk of memory pages has been converted to uncached so now we | ||
130 | * can add it to the pool. | ||
131 | */ | ||
132 | status = gen_pool_add(pool, uc_addr, IA64_GRANULE_SIZE, nid); | ||
133 | if (status) | ||
134 | goto failed; | ||
130 | 135 | ||
131 | allocated_granules++; | 136 | allocated_granules++; |
132 | return addr; | 137 | return 0; |
138 | |||
139 | /* failed to convert or add the chunk so give it back to the kernel */ | ||
140 | failed: | ||
141 | for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++) | ||
142 | ClearPageUncached(&page[i]); | ||
143 | |||
144 | free_pages(c_addr, IA64_GRANULE_SHIFT-PAGE_SHIFT); | ||
145 | return -1; | ||
133 | } | 146 | } |
134 | 147 | ||
135 | 148 | ||
136 | /* | 149 | /* |
137 | * uncached_alloc_page | 150 | * uncached_alloc_page |
138 | * | 151 | * |
152 | * @starting_nid: node id of node to start with, or -1 | ||
153 | * | ||
139 | * Allocate 1 uncached page. Allocates on the requested node. If no | 154 | * Allocate 1 uncached page. Allocates on the requested node. If no |
140 | * uncached pages are available on the requested node, roundrobin starting | 155 | * uncached pages are available on the requested node, roundrobin starting |
141 | * with higher nodes. | 156 | * with the next higher node. |
142 | */ | 157 | */ |
143 | unsigned long | 158 | unsigned long uncached_alloc_page(int starting_nid) |
144 | uncached_alloc_page(int nid) | ||
145 | { | 159 | { |
146 | unsigned long maddr; | 160 | unsigned long uc_addr; |
161 | struct gen_pool *pool; | ||
162 | int nid; | ||
147 | 163 | ||
148 | maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE); | 164 | if (unlikely(starting_nid >= MAX_NUMNODES)) |
165 | return 0; | ||
149 | 166 | ||
150 | dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n", | 167 | if (starting_nid < 0) |
151 | maddr, nid); | 168 | starting_nid = numa_node_id(); |
169 | nid = starting_nid; | ||
152 | 170 | ||
153 | /* | 171 | do { |
154 | * If no memory is availble on our local node, try the | 172 | if (!node_online(nid)) |
155 | * remaining nodes in the system. | 173 | continue; |
156 | */ | 174 | pool = uncached_pool[nid]; |
157 | if (!maddr) { | 175 | if (pool == NULL) |
158 | int i; | 176 | continue; |
159 | 177 | do { | |
160 | for (i = MAX_NUMNODES - 1; i >= 0; i--) { | 178 | uc_addr = gen_pool_alloc(pool, PAGE_SIZE); |
161 | if (i == nid || !node_online(i)) | 179 | if (uc_addr != 0) |
162 | continue; | 180 | return uc_addr; |
163 | maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE); | 181 | } while (uncached_add_chunk(pool, nid) == 0); |
164 | dprintk(KERN_DEBUG "uncached_alloc_page alternate search " | 182 | |
165 | "returns %lx on node %i\n", maddr, i); | 183 | } while ((nid = (nid + 1) % MAX_NUMNODES) != starting_nid); |
166 | if (maddr) { | ||
167 | break; | ||
168 | } | ||
169 | } | ||
170 | } | ||
171 | 184 | ||
172 | return maddr; | 185 | return 0; |
173 | } | 186 | } |
174 | EXPORT_SYMBOL(uncached_alloc_page); | 187 | EXPORT_SYMBOL(uncached_alloc_page); |
175 | 188 | ||
@@ -177,21 +190,22 @@ EXPORT_SYMBOL(uncached_alloc_page); | |||
177 | /* | 190 | /* |
178 | * uncached_free_page | 191 | * uncached_free_page |
179 | * | 192 | * |
193 | * @uc_addr: uncached address of page to free | ||
194 | * | ||
180 | * Free a single uncached page. | 195 | * Free a single uncached page. |
181 | */ | 196 | */ |
182 | void | 197 | void uncached_free_page(unsigned long uc_addr) |
183 | uncached_free_page(unsigned long maddr) | ||
184 | { | 198 | { |
185 | int node; | 199 | int nid = paddr_to_nid(uc_addr - __IA64_UNCACHED_OFFSET); |
186 | 200 | struct gen_pool *pool = uncached_pool[nid]; | |
187 | node = paddr_to_nid(maddr - __IA64_UNCACHED_OFFSET); | ||
188 | 201 | ||
189 | dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node); | 202 | if (unlikely(pool == NULL)) |
203 | return; | ||
190 | 204 | ||
191 | if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET) | 205 | if ((uc_addr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET) |
192 | panic("uncached_free_page invalid address %lx\n", maddr); | 206 | panic("uncached_free_page invalid address %lx\n", uc_addr); |
193 | 207 | ||
194 | gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE); | 208 | gen_pool_free(pool, uc_addr, PAGE_SIZE); |
195 | } | 209 | } |
196 | EXPORT_SYMBOL(uncached_free_page); | 210 | EXPORT_SYMBOL(uncached_free_page); |
197 | 211 | ||
@@ -199,43 +213,39 @@ EXPORT_SYMBOL(uncached_free_page); | |||
199 | /* | 213 | /* |
200 | * uncached_build_memmap, | 214 | * uncached_build_memmap, |
201 | * | 215 | * |
216 | * @uc_start: uncached starting address of a chunk of uncached memory | ||
217 | * @uc_end: uncached ending address of a chunk of uncached memory | ||
218 | * @arg: ignored, (NULL argument passed in on call to efi_memmap_walk_uc()) | ||
219 | * | ||
202 | * Called at boot time to build a map of pages that can be used for | 220 | * Called at boot time to build a map of pages that can be used for |
203 | * memory special operations. | 221 | * memory special operations. |
204 | */ | 222 | */ |
205 | static int __init | 223 | static int __init uncached_build_memmap(unsigned long uc_start, |
206 | uncached_build_memmap(unsigned long start, unsigned long end, void *arg) | 224 | unsigned long uc_end, void *arg) |
207 | { | 225 | { |
208 | long length = end - start; | 226 | int nid = paddr_to_nid(uc_start - __IA64_UNCACHED_OFFSET); |
209 | int node; | 227 | struct gen_pool *pool = uncached_pool[nid]; |
210 | 228 | size_t size = uc_end - uc_start; | |
211 | dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end); | ||
212 | 229 | ||
213 | touch_softlockup_watchdog(); | 230 | touch_softlockup_watchdog(); |
214 | memset((char *)start, 0, length); | ||
215 | 231 | ||
216 | node = paddr_to_nid(start - __IA64_UNCACHED_OFFSET); | 232 | if (pool != NULL) { |
217 | 233 | memset((char *)uc_start, 0, size); | |
218 | for (; start < end ; start += PAGE_SIZE) { | 234 | (void) gen_pool_add(pool, uc_start, size, nid); |
219 | dprintk(KERN_INFO "sticking %lx into the pool!\n", start); | ||
220 | gen_pool_free(uncached_pool[node], start, PAGE_SIZE); | ||
221 | } | 235 | } |
222 | |||
223 | return 0; | 236 | return 0; |
224 | } | 237 | } |
225 | 238 | ||
226 | 239 | ||
227 | static int __init uncached_init(void) { | 240 | static int __init uncached_init(void) |
228 | int i; | 241 | { |
242 | int nid; | ||
229 | 243 | ||
230 | for (i = 0; i < MAX_NUMNODES; i++) { | 244 | for_each_online_node(nid) { |
231 | if (!node_online(i)) | 245 | uncached_pool[nid] = gen_pool_create(PAGE_SHIFT, nid); |
232 | continue; | ||
233 | uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT, | ||
234 | &uncached_get_new_chunk, i); | ||
235 | } | 246 | } |
236 | 247 | ||
237 | efi_memmap_walk_uc(uncached_build_memmap); | 248 | efi_memmap_walk_uc(uncached_build_memmap, NULL); |
238 | |||
239 | return 0; | 249 | return 0; |
240 | } | 250 | } |
241 | 251 | ||