aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/page_alloc.c108
1 files changed, 42 insertions, 66 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a95e72d7f945..418102a02921 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -71,11 +71,6 @@ EXPORT_SYMBOL(nr_swap_pages);
71struct zone *zone_table[1 << (ZONES_SHIFT + NODES_SHIFT)]; 71struct zone *zone_table[1 << (ZONES_SHIFT + NODES_SHIFT)];
72EXPORT_SYMBOL(zone_table); 72EXPORT_SYMBOL(zone_table);
73 73
74#ifdef CONFIG_NUMA
75static struct per_cpu_pageset
76 pageset_table[MAX_NR_ZONES*MAX_NUMNODES*NR_CPUS] __initdata;
77#endif
78
79static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" }; 74static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" };
80int min_free_kbytes = 1024; 75int min_free_kbytes = 1024;
81 76
@@ -652,10 +647,10 @@ static void fastcall free_hot_cold_page(struct page *page, int cold)
652 free_pages_check(__FUNCTION__, page); 647 free_pages_check(__FUNCTION__, page);
653 pcp = &zone_pcp(zone, get_cpu())->pcp[cold]; 648 pcp = &zone_pcp(zone, get_cpu())->pcp[cold];
654 local_irq_save(flags); 649 local_irq_save(flags);
655 if (pcp->count >= pcp->high)
656 pcp->count -= free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
657 list_add(&page->lru, &pcp->list); 650 list_add(&page->lru, &pcp->list);
658 pcp->count++; 651 pcp->count++;
652 if (pcp->count >= pcp->high)
653 pcp->count -= free_pages_bulk(zone, pcp->batch, &pcp->list, 0);
659 local_irq_restore(flags); 654 local_irq_restore(flags);
660 put_cpu(); 655 put_cpu();
661} 656}
@@ -1714,57 +1709,55 @@ static int __devinit zone_batchsize(struct zone *zone)
1714 return batch; 1709 return batch;
1715} 1710}
1716 1711
1712inline void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
1713{
1714 struct per_cpu_pages *pcp;
1715
1716 pcp = &p->pcp[0]; /* hot */
1717 pcp->count = 0;
1718 pcp->low = 2 * batch;
1719 pcp->high = 6 * batch;
1720 pcp->batch = max(1UL, 1 * batch);
1721 INIT_LIST_HEAD(&pcp->list);
1722
1723 pcp = &p->pcp[1]; /* cold*/
1724 pcp->count = 0;
1725 pcp->low = 0;
1726 pcp->high = 2 * batch;
1727 pcp->batch = max(1UL, 1 * batch);
1728 INIT_LIST_HEAD(&pcp->list);
1729}
1730
1717#ifdef CONFIG_NUMA 1731#ifdef CONFIG_NUMA
1718/* 1732/*
1719 * Dynamicaly allocate memory for the 1733 * Boot pageset table. One per cpu which is going to be used for all
1734 * zones and all nodes. The parameters will be set in such a way
1735 * that an item put on a list will immediately be handed over to
1736 * the buddy list. This is safe since pageset manipulation is done
1737 * with interrupts disabled.
1738 *
1739 * Some NUMA counter updates may also be caught by the boot pagesets.
1740 * These will be discarded when bootup is complete.
1741 */
1742static struct per_cpu_pageset
1743 boot_pageset[NR_CPUS] __initdata;
1744
1745/*
1746 * Dynamically allocate memory for the
1720 * per cpu pageset array in struct zone. 1747 * per cpu pageset array in struct zone.
1721 */ 1748 */
1722static int __devinit process_zones(int cpu) 1749static int __devinit process_zones(int cpu)
1723{ 1750{
1724 struct zone *zone, *dzone; 1751 struct zone *zone, *dzone;
1725 int i;
1726 1752
1727 for_each_zone(zone) { 1753 for_each_zone(zone) {
1728 struct per_cpu_pageset *npageset = NULL;
1729 1754
1730 npageset = kmalloc_node(sizeof(struct per_cpu_pageset), 1755 zone->pageset[cpu] = kmalloc_node(sizeof(struct per_cpu_pageset),
1731 GFP_KERNEL, cpu_to_node(cpu)); 1756 GFP_KERNEL, cpu_to_node(cpu));
1732 if (!npageset) { 1757 if (!zone->pageset[cpu])
1733 zone->pageset[cpu] = NULL;
1734 goto bad; 1758 goto bad;
1735 }
1736 1759
1737 if (zone->pageset[cpu]) { 1760 setup_pageset(zone->pageset[cpu], zone_batchsize(zone));
1738 memcpy(npageset, zone->pageset[cpu],
1739 sizeof(struct per_cpu_pageset));
1740
1741 /* Relocate lists */
1742 for (i = 0; i < 2; i++) {
1743 INIT_LIST_HEAD(&npageset->pcp[i].list);
1744 list_splice(&zone->pageset[cpu]->pcp[i].list,
1745 &npageset->pcp[i].list);
1746 }
1747 } else {
1748 struct per_cpu_pages *pcp;
1749 unsigned long batch;
1750
1751 batch = zone_batchsize(zone);
1752
1753 pcp = &npageset->pcp[0]; /* hot */
1754 pcp->count = 0;
1755 pcp->low = 2 * batch;
1756 pcp->high = 6 * batch;
1757 pcp->batch = 1 * batch;
1758 INIT_LIST_HEAD(&pcp->list);
1759
1760 pcp = &npageset->pcp[1]; /* cold*/
1761 pcp->count = 0;
1762 pcp->low = 0;
1763 pcp->high = 2 * batch;
1764 pcp->batch = 1 * batch;
1765 INIT_LIST_HEAD(&pcp->list);
1766 }
1767 zone->pageset[cpu] = npageset;
1768 } 1761 }
1769 1762
1770 return 0; 1763 return 0;
@@ -1878,30 +1871,13 @@ static void __init free_area_init_core(struct pglist_data *pgdat,
1878 batch = zone_batchsize(zone); 1871 batch = zone_batchsize(zone);
1879 1872
1880 for (cpu = 0; cpu < NR_CPUS; cpu++) { 1873 for (cpu = 0; cpu < NR_CPUS; cpu++) {
1881 struct per_cpu_pages *pcp;
1882#ifdef CONFIG_NUMA 1874#ifdef CONFIG_NUMA
1883 struct per_cpu_pageset *pgset; 1875 /* Early boot. Slab allocator not functional yet */
1884 pgset = &pageset_table[nid*MAX_NR_ZONES*NR_CPUS + 1876 zone->pageset[cpu] = &boot_pageset[cpu];
1885 (j * NR_CPUS) + cpu]; 1877 setup_pageset(&boot_pageset[cpu],0);
1886
1887 zone->pageset[cpu] = pgset;
1888#else 1878#else
1889 struct per_cpu_pageset *pgset = zone_pcp(zone, cpu); 1879 setup_pageset(zone_pcp(zone,cpu), batch);
1890#endif 1880#endif
1891
1892 pcp = &pgset->pcp[0]; /* hot */
1893 pcp->count = 0;
1894 pcp->low = 2 * batch;
1895 pcp->high = 6 * batch;
1896 pcp->batch = 1 * batch;
1897 INIT_LIST_HEAD(&pcp->list);
1898
1899 pcp = &pgset->pcp[1]; /* cold */
1900 pcp->count = 0;
1901 pcp->low = 0;
1902 pcp->high = 2 * batch;
1903 pcp->batch = 1 * batch;
1904 INIT_LIST_HEAD(&pcp->list);
1905 } 1881 }
1906 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", 1882 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n",
1907 zone_names[j], realsize, batch); 1883 zone_names[j], realsize, batch);