aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYinghai Lu <yhlu.kernel@gmail.com>2008-03-18 15:44:48 -0400
committerIngo Molnar <mingo@elte.hu>2008-04-26 16:51:08 -0400
commit9a2dc04cf070ee98e014a172695782ff42015fc4 (patch)
tree3a068d5e6625a931f2cf3f7a7133e6f194c962c2
parentad09315cad17458e51c7f1f8b371cb942c54b955 (diff)
mm: offset align in alloc_bootmem()
need offset alignment when node_boot_start's alignment is less than the alignment required. use local node_boot_start to match alignment - so don't add extra operation in search loop. Signed-off-by: Yinghai Lu <yhlu.kernel@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
-rw-r--r--mm/bootmem.c60
1 files changed, 34 insertions, 26 deletions
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 3c012fb58745..0f30bc873ecc 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -206,9 +206,11 @@ void * __init
206__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, 206__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
207 unsigned long align, unsigned long goal, unsigned long limit) 207 unsigned long align, unsigned long goal, unsigned long limit)
208{ 208{
209 unsigned long offset, remaining_size, areasize, preferred; 209 unsigned long areasize, preferred;
210 unsigned long i, start = 0, incr, eidx, end_pfn; 210 unsigned long i, start = 0, incr, eidx, end_pfn;
211 void *ret; 211 void *ret;
212 unsigned long node_boot_start;
213 void *node_bootmem_map;
212 214
213 if (!size) { 215 if (!size) {
214 printk("__alloc_bootmem_core(): zero-sized request\n"); 216 printk("__alloc_bootmem_core(): zero-sized request\n");
@@ -216,23 +218,29 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
216 } 218 }
217 BUG_ON(align & (align-1)); 219 BUG_ON(align & (align-1));
218 220
219 if (limit && bdata->node_boot_start >= limit)
220 return NULL;
221
222 /* on nodes without memory - bootmem_map is NULL */ 221 /* on nodes without memory - bootmem_map is NULL */
223 if (!bdata->node_bootmem_map) 222 if (!bdata->node_bootmem_map)
224 return NULL; 223 return NULL;
225 224
225 /* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
226 node_boot_start = bdata->node_boot_start;
227 node_bootmem_map = bdata->node_bootmem_map;
228 if (align) {
229 node_boot_start = ALIGN(bdata->node_boot_start, align);
230 if (node_boot_start > bdata->node_boot_start)
231 node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
232 PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
233 }
234
235 if (limit && node_boot_start >= limit)
236 return NULL;
237
226 end_pfn = bdata->node_low_pfn; 238 end_pfn = bdata->node_low_pfn;
227 limit = PFN_DOWN(limit); 239 limit = PFN_DOWN(limit);
228 if (limit && end_pfn > limit) 240 if (limit && end_pfn > limit)
229 end_pfn = limit; 241 end_pfn = limit;
230 242
231 eidx = end_pfn - PFN_DOWN(bdata->node_boot_start); 243 eidx = end_pfn - PFN_DOWN(node_boot_start);
232 offset = 0;
233 if (align && (bdata->node_boot_start & (align - 1UL)) != 0)
234 offset = align - (bdata->node_boot_start & (align - 1UL));
235 offset = PFN_DOWN(offset);
236 244
237 /* 245 /*
238 * We try to allocate bootmem pages above 'goal' 246 * We try to allocate bootmem pages above 'goal'
@@ -240,15 +248,16 @@ __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
240 */ 248 */
241 preferred = 0; 249 preferred = 0;
242 if (goal && PFN_DOWN(goal) < end_pfn) { 250 if (goal && PFN_DOWN(goal) < end_pfn) {
243 if (goal > bdata->node_boot_start) 251 if (goal > node_boot_start)
244 preferred = goal - bdata->node_boot_start; 252 preferred = goal - node_boot_start;
245 253
246 if (bdata->last_success >= preferred) 254 if (bdata->last_success > node_boot_start &&
255 bdata->last_success - node_boot_start >= preferred)
247 if (!limit || (limit && limit > bdata->last_success)) 256 if (!limit || (limit && limit > bdata->last_success))
248 preferred = bdata->last_success; 257 preferred = bdata->last_success - node_boot_start;
249 } 258 }
250 259
251 preferred = PFN_DOWN(ALIGN(preferred, align)) + offset; 260 preferred = PFN_DOWN(ALIGN(preferred, align));
252 areasize = (size + PAGE_SIZE-1) / PAGE_SIZE; 261 areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
253 incr = align >> PAGE_SHIFT ? : 1; 262 incr = align >> PAGE_SHIFT ? : 1;
254 263
@@ -256,18 +265,18 @@ restart_scan:
256 for (i = preferred; i < eidx;) { 265 for (i = preferred; i < eidx;) {
257 unsigned long j; 266 unsigned long j;
258 267
259 i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i); 268 i = find_next_zero_bit(node_bootmem_map, eidx, i);
260 i = ALIGN(i, incr); 269 i = ALIGN(i, incr);
261 if (i >= eidx) 270 if (i >= eidx)
262 break; 271 break;
263 if (test_bit(i, bdata->node_bootmem_map)) { 272 if (test_bit(i, node_bootmem_map)) {
264 i += incr; 273 i += incr;
265 continue; 274 continue;
266 } 275 }
267 for (j = i + 1; j < i + areasize; ++j) { 276 for (j = i + 1; j < i + areasize; ++j) {
268 if (j >= eidx) 277 if (j >= eidx)
269 goto fail_block; 278 goto fail_block;
270 if (test_bit(j, bdata->node_bootmem_map)) 279 if (test_bit(j, node_bootmem_map))
271 goto fail_block; 280 goto fail_block;
272 } 281 }
273 start = i; 282 start = i;
@@ -278,14 +287,14 @@ restart_scan:
278 i += incr; 287 i += incr;
279 } 288 }
280 289
281 if (preferred > offset) { 290 if (preferred > 0) {
282 preferred = offset; 291 preferred = 0;
283 goto restart_scan; 292 goto restart_scan;
284 } 293 }
285 return NULL; 294 return NULL;
286 295
287found: 296found:
288 bdata->last_success = PFN_PHYS(start); 297 bdata->last_success = PFN_PHYS(start) + node_boot_start;
289 BUG_ON(start >= eidx); 298 BUG_ON(start >= eidx);
290 299
291 /* 300 /*
@@ -295,6 +304,7 @@ found:
295 */ 304 */
296 if (align < PAGE_SIZE && 305 if (align < PAGE_SIZE &&
297 bdata->last_offset && bdata->last_pos+1 == start) { 306 bdata->last_offset && bdata->last_pos+1 == start) {
307 unsigned long offset, remaining_size;
298 offset = ALIGN(bdata->last_offset, align); 308 offset = ALIGN(bdata->last_offset, align);
299 BUG_ON(offset > PAGE_SIZE); 309 BUG_ON(offset > PAGE_SIZE);
300 remaining_size = PAGE_SIZE - offset; 310 remaining_size = PAGE_SIZE - offset;
@@ -303,14 +313,12 @@ found:
303 /* last_pos unchanged */ 313 /* last_pos unchanged */
304 bdata->last_offset = offset + size; 314 bdata->last_offset = offset + size;
305 ret = phys_to_virt(bdata->last_pos * PAGE_SIZE + 315 ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
306 offset + 316 offset + node_boot_start);
307 bdata->node_boot_start);
308 } else { 317 } else {
309 remaining_size = size - remaining_size; 318 remaining_size = size - remaining_size;
310 areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE; 319 areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
311 ret = phys_to_virt(bdata->last_pos * PAGE_SIZE + 320 ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
312 offset + 321 offset + node_boot_start);
313 bdata->node_boot_start);
314 bdata->last_pos = start + areasize - 1; 322 bdata->last_pos = start + areasize - 1;
315 bdata->last_offset = remaining_size; 323 bdata->last_offset = remaining_size;
316 } 324 }
@@ -318,14 +326,14 @@ found:
318 } else { 326 } else {
319 bdata->last_pos = start + areasize - 1; 327 bdata->last_pos = start + areasize - 1;
320 bdata->last_offset = size & ~PAGE_MASK; 328 bdata->last_offset = size & ~PAGE_MASK;
321 ret = phys_to_virt(start * PAGE_SIZE + bdata->node_boot_start); 329 ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
322 } 330 }
323 331
324 /* 332 /*
325 * Reserve the area now: 333 * Reserve the area now:
326 */ 334 */
327 for (i = start; i < start + areasize; i++) 335 for (i = start; i < start + areasize; i++)
328 if (unlikely(test_and_set_bit(i, bdata->node_bootmem_map))) 336 if (unlikely(test_and_set_bit(i, node_bootmem_map)))
329 BUG(); 337 BUG();
330 memset(ret, 0, size); 338 memset(ret, 0, size);
331 return ret; 339 return ret;