aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/powerpc/Kconfig11
-rw-r--r--arch/powerpc/mm/numa.c365
-rw-r--r--arch/ppc64/Kconfig11
-rw-r--r--include/asm-powerpc/topology.h10
-rw-r--r--include/asm-ppc64/mmzone.h63
-rw-r--r--include/asm-ppc64/page.h5
6 files changed, 169 insertions, 296 deletions
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index ed31062029f7..c523029674e6 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -581,17 +581,12 @@ config ARCH_FLATMEM_ENABLE
581 def_bool y 581 def_bool y
582 depends on PPC64 && !NUMA 582 depends on PPC64 && !NUMA
583 583
584config ARCH_DISCONTIGMEM_ENABLE 584config ARCH_SPARSEMEM_ENABLE
585 def_bool y
586 depends on SMP && PPC_PSERIES
587
588config ARCH_DISCONTIGMEM_DEFAULT
589 def_bool y 585 def_bool y
590 depends on ARCH_DISCONTIGMEM_ENABLE
591 586
592config ARCH_SPARSEMEM_ENABLE 587config ARCH_SPARSEMEM_DEFAULT
593 def_bool y 588 def_bool y
594 depends on ARCH_DISCONTIGMEM_ENABLE 589 depends on SMP && PPC_PSERIES
595 590
596source "mm/Kconfig" 591source "mm/Kconfig"
597 592
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index c2d7fec50c92..bd2cf1336885 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -17,9 +17,8 @@
17#include <linux/nodemask.h> 17#include <linux/nodemask.h>
18#include <linux/cpu.h> 18#include <linux/cpu.h>
19#include <linux/notifier.h> 19#include <linux/notifier.h>
20#include <asm/sparsemem.h>
20#include <asm/lmb.h> 21#include <asm/lmb.h>
21#include <asm/machdep.h>
22#include <asm/abs_addr.h>
23#include <asm/system.h> 22#include <asm/system.h>
24#include <asm/smp.h> 23#include <asm/smp.h>
25 24
@@ -28,42 +27,113 @@ static int numa_enabled = 1;
28static int numa_debug; 27static int numa_debug;
29#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); } 28#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
30 29
31#ifdef DEBUG_NUMA 30int numa_cpu_lookup_table[NR_CPUS];
32#define ARRAY_INITIALISER -1
33#else
34#define ARRAY_INITIALISER 0
35#endif
36
37int numa_cpu_lookup_table[NR_CPUS] = { [ 0 ... (NR_CPUS - 1)] =
38 ARRAY_INITIALISER};
39char *numa_memory_lookup_table;
40cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; 31cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
41
42struct pglist_data *node_data[MAX_NUMNODES]; 32struct pglist_data *node_data[MAX_NUMNODES];
43bootmem_data_t __initdata plat_node_bdata[MAX_NUMNODES]; 33
34EXPORT_SYMBOL(numa_cpu_lookup_table);
35EXPORT_SYMBOL(numa_cpumask_lookup_table);
36EXPORT_SYMBOL(node_data);
37
38static bootmem_data_t __initdata plat_node_bdata[MAX_NUMNODES];
44static int min_common_depth; 39static int min_common_depth;
45 40
46/* 41/*
47 * We need somewhere to store start/span for each node until we have 42 * We need somewhere to store start/end/node for each region until we have
48 * allocated the real node_data structures. 43 * allocated the real node_data structures.
49 */ 44 */
45#define MAX_REGIONS (MAX_LMB_REGIONS*2)
50static struct { 46static struct {
51 unsigned long node_start_pfn; 47 unsigned long start_pfn;
52 unsigned long node_end_pfn; 48 unsigned long end_pfn;
53 unsigned long node_present_pages; 49 int nid;
54} init_node_data[MAX_NUMNODES] __initdata; 50} init_node_data[MAX_REGIONS] __initdata;
55 51
56EXPORT_SYMBOL(node_data); 52int __init early_pfn_to_nid(unsigned long pfn)
57EXPORT_SYMBOL(numa_cpu_lookup_table); 53{
58EXPORT_SYMBOL(numa_memory_lookup_table); 54 unsigned int i;
59EXPORT_SYMBOL(numa_cpumask_lookup_table); 55
56 for (i = 0; init_node_data[i].end_pfn; i++) {
57 unsigned long start_pfn = init_node_data[i].start_pfn;
58 unsigned long end_pfn = init_node_data[i].end_pfn;
59
60 if ((start_pfn <= pfn) && (pfn < end_pfn))
61 return init_node_data[i].nid;
62 }
63
64 return -1;
65}
66
67void __init add_region(unsigned int nid, unsigned long start_pfn,
68 unsigned long pages)
69{
70 unsigned int i;
71
72 dbg("add_region nid %d start_pfn 0x%lx pages 0x%lx\n",
73 nid, start_pfn, pages);
74
75 for (i = 0; init_node_data[i].end_pfn; i++) {
76 if (init_node_data[i].nid != nid)
77 continue;
78 if (init_node_data[i].end_pfn == start_pfn) {
79 init_node_data[i].end_pfn += pages;
80 return;
81 }
82 if (init_node_data[i].start_pfn == (start_pfn + pages)) {
83 init_node_data[i].start_pfn -= pages;
84 return;
85 }
86 }
87
88 /*
89 * Leave last entry NULL so we dont iterate off the end (we use
90 * entry.end_pfn to terminate the walk).
91 */
92 if (i >= (MAX_REGIONS - 1)) {
93 printk(KERN_ERR "WARNING: too many memory regions in "
94 "numa code, truncating\n");
95 return;
96 }
97
98 init_node_data[i].start_pfn = start_pfn;
99 init_node_data[i].end_pfn = start_pfn + pages;
100 init_node_data[i].nid = nid;
101}
102
103/* We assume init_node_data has no overlapping regions */
104void __init get_region(unsigned int nid, unsigned long *start_pfn,
105 unsigned long *end_pfn, unsigned long *pages_present)
106{
107 unsigned int i;
108
109 *start_pfn = -1UL;
110 *end_pfn = *pages_present = 0;
111
112 for (i = 0; init_node_data[i].end_pfn; i++) {
113 if (init_node_data[i].nid != nid)
114 continue;
115
116 *pages_present += init_node_data[i].end_pfn -
117 init_node_data[i].start_pfn;
118
119 if (init_node_data[i].start_pfn < *start_pfn)
120 *start_pfn = init_node_data[i].start_pfn;
121
122 if (init_node_data[i].end_pfn > *end_pfn)
123 *end_pfn = init_node_data[i].end_pfn;
124 }
125
126 /* We didnt find a matching region, return start/end as 0 */
127 if (*start_pfn == -1UL)
128 start_pfn = 0;
129}
60 130
61static inline void map_cpu_to_node(int cpu, int node) 131static inline void map_cpu_to_node(int cpu, int node)
62{ 132{
63 numa_cpu_lookup_table[cpu] = node; 133 numa_cpu_lookup_table[cpu] = node;
64 if (!(cpu_isset(cpu, numa_cpumask_lookup_table[node]))) { 134
135 if (!(cpu_isset(cpu, numa_cpumask_lookup_table[node])))
65 cpu_set(cpu, numa_cpumask_lookup_table[node]); 136 cpu_set(cpu, numa_cpumask_lookup_table[node]);
66 }
67} 137}
68 138
69#ifdef CONFIG_HOTPLUG_CPU 139#ifdef CONFIG_HOTPLUG_CPU
@@ -82,7 +152,7 @@ static void unmap_cpu_from_node(unsigned long cpu)
82} 152}
83#endif /* CONFIG_HOTPLUG_CPU */ 153#endif /* CONFIG_HOTPLUG_CPU */
84 154
85static struct device_node * __devinit find_cpu_node(unsigned int cpu) 155static struct device_node *find_cpu_node(unsigned int cpu)
86{ 156{
87 unsigned int hw_cpuid = get_hard_smp_processor_id(cpu); 157 unsigned int hw_cpuid = get_hard_smp_processor_id(cpu);
88 struct device_node *cpu_node = NULL; 158 struct device_node *cpu_node = NULL;
@@ -209,7 +279,7 @@ static int __init get_mem_size_cells(void)
209 return rc; 279 return rc;
210} 280}
211 281
212static unsigned long read_n_cells(int n, unsigned int **buf) 282static unsigned long __init read_n_cells(int n, unsigned int **buf)
213{ 283{
214 unsigned long result = 0; 284 unsigned long result = 0;
215 285
@@ -291,7 +361,8 @@ static int cpu_numa_callback(struct notifier_block *nfb,
291 * or zero. If the returned value of size is 0 the region should be 361 * or zero. If the returned value of size is 0 the region should be
292 * discarded as it lies wholy above the memory limit. 362 * discarded as it lies wholy above the memory limit.
293 */ 363 */
294static unsigned long __init numa_enforce_memory_limit(unsigned long start, unsigned long size) 364static unsigned long __init numa_enforce_memory_limit(unsigned long start,
365 unsigned long size)
295{ 366{
296 /* 367 /*
297 * We use lmb_end_of_DRAM() in here instead of memory_limit because 368 * We use lmb_end_of_DRAM() in here instead of memory_limit because
@@ -316,8 +387,7 @@ static int __init parse_numa_properties(void)
316 struct device_node *cpu = NULL; 387 struct device_node *cpu = NULL;
317 struct device_node *memory = NULL; 388 struct device_node *memory = NULL;
318 int addr_cells, size_cells; 389 int addr_cells, size_cells;
319 int max_domain = 0; 390 int max_domain;
320 long entries = lmb_end_of_DRAM() >> MEMORY_INCREMENT_SHIFT;
321 unsigned long i; 391 unsigned long i;
322 392
323 if (numa_enabled == 0) { 393 if (numa_enabled == 0) {
@@ -325,13 +395,6 @@ static int __init parse_numa_properties(void)
325 return -1; 395 return -1;
326 } 396 }
327 397
328 numa_memory_lookup_table =
329 (char *)abs_to_virt(lmb_alloc(entries * sizeof(char), 1));
330 memset(numa_memory_lookup_table, 0, entries * sizeof(char));
331
332 for (i = 0; i < entries ; i++)
333 numa_memory_lookup_table[i] = ARRAY_INITIALISER;
334
335 min_common_depth = find_min_common_depth(); 398 min_common_depth = find_min_common_depth();
336 399
337 dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth); 400 dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);
@@ -383,9 +446,6 @@ new_range:
383 start = read_n_cells(addr_cells, &memcell_buf); 446 start = read_n_cells(addr_cells, &memcell_buf);
384 size = read_n_cells(size_cells, &memcell_buf); 447 size = read_n_cells(size_cells, &memcell_buf);
385 448
386 start = _ALIGN_DOWN(start, MEMORY_INCREMENT);
387 size = _ALIGN_UP(size, MEMORY_INCREMENT);
388
389 numa_domain = of_node_numa_domain(memory); 449 numa_domain = of_node_numa_domain(memory);
390 450
391 if (numa_domain >= MAX_NUMNODES) { 451 if (numa_domain >= MAX_NUMNODES) {
@@ -399,44 +459,15 @@ new_range:
399 if (max_domain < numa_domain) 459 if (max_domain < numa_domain)
400 max_domain = numa_domain; 460 max_domain = numa_domain;
401 461
402 if (! (size = numa_enforce_memory_limit(start, size))) { 462 if (!(size = numa_enforce_memory_limit(start, size))) {
403 if (--ranges) 463 if (--ranges)
404 goto new_range; 464 goto new_range;
405 else 465 else
406 continue; 466 continue;
407 } 467 }
408 468
409 /* 469 add_region(numa_domain, start >> PAGE_SHIFT,
410 * Initialize new node struct, or add to an existing one. 470 size >> PAGE_SHIFT);
411 */
412 if (init_node_data[numa_domain].node_end_pfn) {
413 if ((start / PAGE_SIZE) <
414 init_node_data[numa_domain].node_start_pfn)
415 init_node_data[numa_domain].node_start_pfn =
416 start / PAGE_SIZE;
417 if (((start / PAGE_SIZE) + (size / PAGE_SIZE)) >
418 init_node_data[numa_domain].node_end_pfn)
419 init_node_data[numa_domain].node_end_pfn =
420 (start / PAGE_SIZE) +
421 (size / PAGE_SIZE);
422
423 init_node_data[numa_domain].node_present_pages +=
424 size / PAGE_SIZE;
425 } else {
426 node_set_online(numa_domain);
427
428 init_node_data[numa_domain].node_start_pfn =
429 start / PAGE_SIZE;
430 init_node_data[numa_domain].node_end_pfn =
431 init_node_data[numa_domain].node_start_pfn +
432 size / PAGE_SIZE;
433 init_node_data[numa_domain].node_present_pages =
434 size / PAGE_SIZE;
435 }
436
437 for (i = start ; i < (start+size); i += MEMORY_INCREMENT)
438 numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] =
439 numa_domain;
440 471
441 if (--ranges) 472 if (--ranges)
442 goto new_range; 473 goto new_range;
@@ -452,32 +483,15 @@ static void __init setup_nonnuma(void)
452{ 483{
453 unsigned long top_of_ram = lmb_end_of_DRAM(); 484 unsigned long top_of_ram = lmb_end_of_DRAM();
454 unsigned long total_ram = lmb_phys_mem_size(); 485 unsigned long total_ram = lmb_phys_mem_size();
455 unsigned long i;
456 486
457 printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", 487 printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
458 top_of_ram, total_ram); 488 top_of_ram, total_ram);
459 printk(KERN_INFO "Memory hole size: %ldMB\n", 489 printk(KERN_INFO "Memory hole size: %ldMB\n",
460 (top_of_ram - total_ram) >> 20); 490 (top_of_ram - total_ram) >> 20);
461 491
462 if (!numa_memory_lookup_table) {
463 long entries = top_of_ram >> MEMORY_INCREMENT_SHIFT;
464 numa_memory_lookup_table =
465 (char *)abs_to_virt(lmb_alloc(entries * sizeof(char), 1));
466 memset(numa_memory_lookup_table, 0, entries * sizeof(char));
467 for (i = 0; i < entries ; i++)
468 numa_memory_lookup_table[i] = ARRAY_INITIALISER;
469 }
470
471 map_cpu_to_node(boot_cpuid, 0); 492 map_cpu_to_node(boot_cpuid, 0);
472 493 add_region(0, 0, lmb_end_of_DRAM() >> PAGE_SHIFT);
473 node_set_online(0); 494 node_set_online(0);
474
475 init_node_data[0].node_start_pfn = 0;
476 init_node_data[0].node_end_pfn = lmb_end_of_DRAM() / PAGE_SIZE;
477 init_node_data[0].node_present_pages = total_ram / PAGE_SIZE;
478
479 for (i = 0 ; i < top_of_ram; i += MEMORY_INCREMENT)
480 numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] = 0;
481} 495}
482 496
483static void __init dump_numa_topology(void) 497static void __init dump_numa_topology(void)
@@ -495,8 +509,9 @@ static void __init dump_numa_topology(void)
495 509
496 count = 0; 510 count = 0;
497 511
498 for (i = 0; i < lmb_end_of_DRAM(); i += MEMORY_INCREMENT) { 512 for (i = 0; i < lmb_end_of_DRAM();
499 if (numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] == node) { 513 i += (1 << SECTION_SIZE_BITS)) {
514 if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) {
500 if (count == 0) 515 if (count == 0)
501 printk(" 0x%lx", i); 516 printk(" 0x%lx", i);
502 ++count; 517 ++count;
@@ -521,10 +536,12 @@ static void __init dump_numa_topology(void)
521 * 536 *
522 * Returns the physical address of the memory. 537 * Returns the physical address of the memory.
523 */ 538 */
524static unsigned long careful_allocation(int nid, unsigned long size, 539static void __init *careful_allocation(int nid, unsigned long size,
525 unsigned long align, unsigned long end) 540 unsigned long align,
541 unsigned long end_pfn)
526{ 542{
527 unsigned long ret = lmb_alloc_base(size, align, end); 543 int new_nid;
544 unsigned long ret = lmb_alloc_base(size, align, end_pfn << PAGE_SHIFT);
528 545
529 /* retry over all memory */ 546 /* retry over all memory */
530 if (!ret) 547 if (!ret)
@@ -538,28 +555,27 @@ static unsigned long careful_allocation(int nid, unsigned long size,
538 * If the memory came from a previously allocated node, we must 555 * If the memory came from a previously allocated node, we must
539 * retry with the bootmem allocator. 556 * retry with the bootmem allocator.
540 */ 557 */
541 if (pa_to_nid(ret) < nid) { 558 new_nid = early_pfn_to_nid(ret >> PAGE_SHIFT);
542 nid = pa_to_nid(ret); 559 if (new_nid < nid) {
543 ret = (unsigned long)__alloc_bootmem_node(NODE_DATA(nid), 560 ret = (unsigned long)__alloc_bootmem_node(NODE_DATA(new_nid),
544 size, align, 0); 561 size, align, 0);
545 562
546 if (!ret) 563 if (!ret)
547 panic("numa.c: cannot allocate %lu bytes on node %d", 564 panic("numa.c: cannot allocate %lu bytes on node %d",
548 size, nid); 565 size, new_nid);
549 566
550 ret = virt_to_abs(ret); 567 ret = __pa(ret);
551 568
552 dbg("alloc_bootmem %lx %lx\n", ret, size); 569 dbg("alloc_bootmem %lx %lx\n", ret, size);
553 } 570 }
554 571
555 return ret; 572 return (void *)ret;
556} 573}
557 574
558void __init do_init_bootmem(void) 575void __init do_init_bootmem(void)
559{ 576{
560 int nid; 577 int nid;
561 int addr_cells, size_cells; 578 unsigned int i;
562 struct device_node *memory = NULL;
563 static struct notifier_block ppc64_numa_nb = { 579 static struct notifier_block ppc64_numa_nb = {
564 .notifier_call = cpu_numa_callback, 580 .notifier_call = cpu_numa_callback,
565 .priority = 1 /* Must run before sched domains notifier. */ 581 .priority = 1 /* Must run before sched domains notifier. */
@@ -577,99 +593,66 @@ void __init do_init_bootmem(void)
577 register_cpu_notifier(&ppc64_numa_nb); 593 register_cpu_notifier(&ppc64_numa_nb);
578 594
579 for_each_online_node(nid) { 595 for_each_online_node(nid) {
580 unsigned long start_paddr, end_paddr; 596 unsigned long start_pfn, end_pfn, pages_present;
581 int i;
582 unsigned long bootmem_paddr; 597 unsigned long bootmem_paddr;
583 unsigned long bootmap_pages; 598 unsigned long bootmap_pages;
584 599
585 start_paddr = init_node_data[nid].node_start_pfn * PAGE_SIZE; 600 get_region(nid, &start_pfn, &end_pfn, &pages_present);
586 end_paddr = init_node_data[nid].node_end_pfn * PAGE_SIZE;
587 601
588 /* Allocate the node structure node local if possible */ 602 /* Allocate the node structure node local if possible */
589 NODE_DATA(nid) = (struct pglist_data *)careful_allocation(nid, 603 NODE_DATA(nid) = careful_allocation(nid,
590 sizeof(struct pglist_data), 604 sizeof(struct pglist_data),
591 SMP_CACHE_BYTES, end_paddr); 605 SMP_CACHE_BYTES, end_pfn);
592 NODE_DATA(nid) = abs_to_virt(NODE_DATA(nid)); 606 NODE_DATA(nid) = __va(NODE_DATA(nid));
593 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); 607 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
594 608
595 dbg("node %d\n", nid); 609 dbg("node %d\n", nid);
596 dbg("NODE_DATA() = %p\n", NODE_DATA(nid)); 610 dbg("NODE_DATA() = %p\n", NODE_DATA(nid));
597 611
598 NODE_DATA(nid)->bdata = &plat_node_bdata[nid]; 612 NODE_DATA(nid)->bdata = &plat_node_bdata[nid];
599 NODE_DATA(nid)->node_start_pfn = 613 NODE_DATA(nid)->node_start_pfn = start_pfn;
600 init_node_data[nid].node_start_pfn; 614 NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
601 NODE_DATA(nid)->node_spanned_pages =
602 end_paddr - start_paddr;
603 615
604 if (NODE_DATA(nid)->node_spanned_pages == 0) 616 if (NODE_DATA(nid)->node_spanned_pages == 0)
605 continue; 617 continue;
606 618
607 dbg("start_paddr = %lx\n", start_paddr); 619 dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT);
608 dbg("end_paddr = %lx\n", end_paddr); 620 dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT);
609 621
610 bootmap_pages = bootmem_bootmap_pages((end_paddr - start_paddr) >> PAGE_SHIFT); 622 bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
623 bootmem_paddr = (unsigned long)careful_allocation(nid,
624 bootmap_pages << PAGE_SHIFT,
625 PAGE_SIZE, end_pfn);
626 memset(__va(bootmem_paddr), 0, bootmap_pages << PAGE_SHIFT);
611 627
612 bootmem_paddr = careful_allocation(nid,
613 bootmap_pages << PAGE_SHIFT,
614 PAGE_SIZE, end_paddr);
615 memset(abs_to_virt(bootmem_paddr), 0,
616 bootmap_pages << PAGE_SHIFT);
617 dbg("bootmap_paddr = %lx\n", bootmem_paddr); 628 dbg("bootmap_paddr = %lx\n", bootmem_paddr);
618 629
619 init_bootmem_node(NODE_DATA(nid), bootmem_paddr >> PAGE_SHIFT, 630 init_bootmem_node(NODE_DATA(nid), bootmem_paddr >> PAGE_SHIFT,
620 start_paddr >> PAGE_SHIFT, 631 start_pfn, end_pfn);
621 end_paddr >> PAGE_SHIFT);
622 632
623 /* 633 /* Add free regions on this node */
624 * We need to do another scan of all memory sections to 634 for (i = 0; init_node_data[i].end_pfn; i++) {
625 * associate memory with the correct node. 635 unsigned long start, end;
626 */
627 addr_cells = get_mem_addr_cells();
628 size_cells = get_mem_size_cells();
629 memory = NULL;
630 while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
631 unsigned long mem_start, mem_size;
632 int numa_domain, ranges;
633 unsigned int *memcell_buf;
634 unsigned int len;
635
636 memcell_buf = (unsigned int *)get_property(memory, "reg", &len);
637 if (!memcell_buf || len <= 0)
638 continue;
639 636
640 ranges = memory->n_addrs; /* ranges in cell */ 637 if (init_node_data[i].nid != nid)
641new_range:
642 mem_start = read_n_cells(addr_cells, &memcell_buf);
643 mem_size = read_n_cells(size_cells, &memcell_buf);
644 if (numa_enabled) {
645 numa_domain = of_node_numa_domain(memory);
646 if (numa_domain >= MAX_NUMNODES)
647 numa_domain = 0;
648 } else
649 numa_domain = 0;
650
651 if (numa_domain != nid)
652 continue; 638 continue;
653 639
654 mem_size = numa_enforce_memory_limit(mem_start, mem_size); 640 start = init_node_data[i].start_pfn << PAGE_SHIFT;
655 if (mem_size) { 641 end = init_node_data[i].end_pfn << PAGE_SHIFT;
656 dbg("free_bootmem %lx %lx\n", mem_start, mem_size);
657 free_bootmem_node(NODE_DATA(nid), mem_start, mem_size);
658 }
659 642
660 if (--ranges) /* process all ranges in cell */ 643 dbg("free_bootmem %lx %lx\n", start, end - start);
661 goto new_range; 644 free_bootmem_node(NODE_DATA(nid), start, end - start);
662 } 645 }
663 646
664 /* 647 /* Mark reserved regions on this node */
665 * Mark reserved regions on this node
666 */
667 for (i = 0; i < lmb.reserved.cnt; i++) { 648 for (i = 0; i < lmb.reserved.cnt; i++) {
668 unsigned long physbase = lmb.reserved.region[i].base; 649 unsigned long physbase = lmb.reserved.region[i].base;
669 unsigned long size = lmb.reserved.region[i].size; 650 unsigned long size = lmb.reserved.region[i].size;
651 unsigned long start_paddr = start_pfn << PAGE_SHIFT;
652 unsigned long end_paddr = end_pfn << PAGE_SHIFT;
670 653
671 if (pa_to_nid(physbase) != nid && 654 if (early_pfn_to_nid(physbase >> PAGE_SHIFT) != nid &&
672 pa_to_nid(physbase+size-1) != nid) 655 early_pfn_to_nid((physbase+size-1) >> PAGE_SHIFT) != nid)
673 continue; 656 continue;
674 657
675 if (physbase < end_paddr && 658 if (physbase < end_paddr &&
@@ -689,46 +672,19 @@ new_range:
689 size); 672 size);
690 } 673 }
691 } 674 }
692 /*
693 * This loop may look famaliar, but we have to do it again
694 * after marking our reserved memory to mark memory present
695 * for sparsemem.
696 */
697 addr_cells = get_mem_addr_cells();
698 size_cells = get_mem_size_cells();
699 memory = NULL;
700 while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
701 unsigned long mem_start, mem_size;
702 int numa_domain, ranges;
703 unsigned int *memcell_buf;
704 unsigned int len;
705
706 memcell_buf = (unsigned int *)get_property(memory, "reg", &len);
707 if (!memcell_buf || len <= 0)
708 continue;
709 675
710 ranges = memory->n_addrs; /* ranges in cell */ 676 /* Add regions into sparsemem */
711new_range2: 677 for (i = 0; init_node_data[i].end_pfn; i++) {
712 mem_start = read_n_cells(addr_cells, &memcell_buf); 678 unsigned long start, end;
713 mem_size = read_n_cells(size_cells, &memcell_buf); 679
714 if (numa_enabled) { 680 if (init_node_data[i].nid != nid)
715 numa_domain = of_node_numa_domain(memory);
716 if (numa_domain >= MAX_NUMNODES)
717 numa_domain = 0;
718 } else
719 numa_domain = 0;
720
721 if (numa_domain != nid)
722 continue; 681 continue;
723 682
724 mem_size = numa_enforce_memory_limit(mem_start, mem_size); 683 start = init_node_data[i].start_pfn;
725 memory_present(numa_domain, mem_start >> PAGE_SHIFT, 684 end = init_node_data[i].end_pfn;
726 (mem_start + mem_size) >> PAGE_SHIFT);
727 685
728 if (--ranges) /* process all ranges in cell */ 686 memory_present(nid, start, end);
729 goto new_range2;
730 } 687 }
731
732 } 688 }
733} 689}
734 690
@@ -742,21 +698,18 @@ void __init paging_init(void)
742 memset(zholes_size, 0, sizeof(zholes_size)); 698 memset(zholes_size, 0, sizeof(zholes_size));
743 699
744 for_each_online_node(nid) { 700 for_each_online_node(nid) {
745 unsigned long start_pfn; 701 unsigned long start_pfn, end_pfn, pages_present;
746 unsigned long end_pfn;
747 702
748 start_pfn = init_node_data[nid].node_start_pfn; 703 get_region(nid, &start_pfn, &end_pfn, &pages_present);
749 end_pfn = init_node_data[nid].node_end_pfn;
750 704
751 zones_size[ZONE_DMA] = end_pfn - start_pfn; 705 zones_size[ZONE_DMA] = end_pfn - start_pfn;
752 zholes_size[ZONE_DMA] = zones_size[ZONE_DMA] - 706 zholes_size[ZONE_DMA] = zones_size[ZONE_DMA] - pages_present;
753 init_node_data[nid].node_present_pages;
754 707
755 dbg("free_area_init node %d %lx %lx (hole: %lx)\n", nid, 708 dbg("free_area_init node %d %lx %lx (hole: %lx)\n", nid,
756 zones_size[ZONE_DMA], start_pfn, zholes_size[ZONE_DMA]); 709 zones_size[ZONE_DMA], start_pfn, zholes_size[ZONE_DMA]);
757 710
758 free_area_init_node(nid, NODE_DATA(nid), zones_size, 711 free_area_init_node(nid, NODE_DATA(nid), zones_size, start_pfn,
759 start_pfn, zholes_size); 712 zholes_size);
760 } 713 }
761} 714}
762 715
diff --git a/arch/ppc64/Kconfig b/arch/ppc64/Kconfig
index c9d32db9d76a..9d10c12e87fe 100644
--- a/arch/ppc64/Kconfig
+++ b/arch/ppc64/Kconfig
@@ -279,17 +279,12 @@ config ARCH_FLATMEM_ENABLE
279 def_bool y 279 def_bool y
280 depends on !NUMA 280 depends on !NUMA
281 281
282config ARCH_DISCONTIGMEM_ENABLE 282config ARCH_SPARSEMEM_ENABLE
283 def_bool y
284 depends on SMP && PPC_PSERIES
285
286config ARCH_DISCONTIGMEM_DEFAULT
287 def_bool y 283 def_bool y
288 depends on ARCH_DISCONTIGMEM_ENABLE
289 284
290config ARCH_SPARSEMEM_ENABLE 285config ARCH_SPARSEMEM_DEFAULT
291 def_bool y 286 def_bool y
292 depends on ARCH_DISCONTIGMEM_ENABLE 287 depends on NUMA
293 288
294source "mm/Kconfig" 289source "mm/Kconfig"
295 290
diff --git a/include/asm-powerpc/topology.h b/include/asm-powerpc/topology.h
index 388cc9f1277f..015d28746e1b 100644
--- a/include/asm-powerpc/topology.h
+++ b/include/asm-powerpc/topology.h
@@ -9,15 +9,7 @@
9 9
10static inline int cpu_to_node(int cpu) 10static inline int cpu_to_node(int cpu)
11{ 11{
12 int node; 12 return numa_cpu_lookup_table[cpu];
13
14 node = numa_cpu_lookup_table[cpu];
15
16#ifdef DEBUG_NUMA
17 BUG_ON(node == -1);
18#endif
19
20 return node;
21} 13}
22 14
23#define parent_node(node) (node) 15#define parent_node(node) (node)
diff --git a/include/asm-ppc64/mmzone.h b/include/asm-ppc64/mmzone.h
index 7f06afbfbabb..54958d6cae04 100644
--- a/include/asm-ppc64/mmzone.h
+++ b/include/asm-ppc64/mmzone.h
@@ -8,15 +8,14 @@
8#define _ASM_MMZONE_H_ 8#define _ASM_MMZONE_H_
9 9
10#include <linux/config.h> 10#include <linux/config.h>
11#include <asm/smp.h>
12 11
13/* generic non-linear memory support: 12/*
13 * generic non-linear memory support:
14 * 14 *
15 * 1) we will not split memory into more chunks than will fit into the 15 * 1) we will not split memory into more chunks than will fit into the
16 * flags field of the struct page 16 * flags field of the struct page
17 */ 17 */
18 18
19
20#ifdef CONFIG_NEED_MULTIPLE_NODES 19#ifdef CONFIG_NEED_MULTIPLE_NODES
21 20
22extern struct pglist_data *node_data[]; 21extern struct pglist_data *node_data[];
@@ -30,41 +29,11 @@ extern struct pglist_data *node_data[];
30 */ 29 */
31 30
32extern int numa_cpu_lookup_table[]; 31extern int numa_cpu_lookup_table[];
33extern char *numa_memory_lookup_table;
34extern cpumask_t numa_cpumask_lookup_table[]; 32extern cpumask_t numa_cpumask_lookup_table[];
35#ifdef CONFIG_MEMORY_HOTPLUG 33#ifdef CONFIG_MEMORY_HOTPLUG
36extern unsigned long max_pfn; 34extern unsigned long max_pfn;
37#endif 35#endif
38 36
39/* 16MB regions */
40#define MEMORY_INCREMENT_SHIFT 24
41#define MEMORY_INCREMENT (1UL << MEMORY_INCREMENT_SHIFT)
42
43/* NUMA debugging, will not work on a DLPAR machine */
44#undef DEBUG_NUMA
45
46static inline int pa_to_nid(unsigned long pa)
47{
48 int nid;
49
50#ifdef CONFIG_MEMORY_HOTPLUG
51 /* kludge hot added sections default to node 0 */
52 if (pa >= (max_pfn << PAGE_SHIFT))
53 return 0;
54#endif
55 nid = numa_memory_lookup_table[pa >> MEMORY_INCREMENT_SHIFT];
56
57#ifdef DEBUG_NUMA
58 /* the physical address passed in is not in the map for the system */
59 if (nid == -1) {
60 printk("bad address: %lx\n", pa);
61 BUG();
62 }
63#endif
64
65 return nid;
66}
67
68/* 37/*
69 * Following are macros that each numa implmentation must define. 38 * Following are macros that each numa implmentation must define.
70 */ 39 */
@@ -72,36 +41,10 @@ static inline int pa_to_nid(unsigned long pa)
72#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) 41#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
73#define node_end_pfn(nid) (NODE_DATA(nid)->node_end_pfn) 42#define node_end_pfn(nid) (NODE_DATA(nid)->node_end_pfn)
74 43
75#ifdef CONFIG_DISCONTIGMEM
76
77#define node_localnr(pfn, nid) ((pfn) - NODE_DATA(nid)->node_start_pfn)
78
79#define pfn_to_nid(pfn) pa_to_nid((unsigned long)(pfn) << PAGE_SHIFT)
80
81/* Written this way to avoid evaluating arguments twice */
82#define discontigmem_pfn_to_page(pfn) \
83({ \
84 unsigned long __tmp = pfn; \
85 (NODE_DATA(pfn_to_nid(__tmp))->node_mem_map + \
86 node_localnr(__tmp, pfn_to_nid(__tmp))); \
87})
88
89#define discontigmem_page_to_pfn(p) \
90({ \
91 struct page *__tmp = p; \
92 (((__tmp) - page_zone(__tmp)->zone_mem_map) + \
93 page_zone(__tmp)->zone_start_pfn); \
94})
95
96/* XXX fix for discontiguous physical memory */
97#define discontigmem_pfn_valid(pfn) ((pfn) < num_physpages)
98
99#endif /* CONFIG_DISCONTIGMEM */
100
101#endif /* CONFIG_NEED_MULTIPLE_NODES */ 44#endif /* CONFIG_NEED_MULTIPLE_NODES */
102 45
103#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID 46#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
104#define early_pfn_to_nid(pfn) pa_to_nid(((unsigned long)pfn) << PAGE_SHIFT) 47extern int __init early_pfn_to_nid(unsigned long pfn);
105#endif 48#endif
106 49
107#endif /* _ASM_MMZONE_H_ */ 50#endif /* _ASM_MMZONE_H_ */
diff --git a/include/asm-ppc64/page.h b/include/asm-ppc64/page.h
index e32f1187aa29..3efc3288f7e9 100644
--- a/include/asm-ppc64/page.h
+++ b/include/asm-ppc64/page.h
@@ -279,11 +279,6 @@ extern u64 ppc64_pft_size; /* Log 2 of page table size */
279 279
280#define __va(x) ((void *)((unsigned long)(x) + KERNELBASE)) 280#define __va(x) ((void *)((unsigned long)(x) + KERNELBASE))
281 281
282#ifdef CONFIG_DISCONTIGMEM
283#define page_to_pfn(page) discontigmem_page_to_pfn(page)
284#define pfn_to_page(pfn) discontigmem_pfn_to_page(pfn)
285#define pfn_valid(pfn) discontigmem_pfn_valid(pfn)
286#endif
287#ifdef CONFIG_FLATMEM 282#ifdef CONFIG_FLATMEM
288#define pfn_to_page(pfn) (mem_map + (pfn)) 283#define pfn_to_page(pfn) (mem_map + (pfn))
289#define page_to_pfn(page) ((unsigned long)((page) - mem_map)) 284#define page_to_pfn(page) ((unsigned long)((page) - mem_map))